"""This tutorial introduces restricted boltzmann machines (RBM) using Theano.

Boltzmann Machines (BMs) are a particular form of energy-based model which

contain hidden variables. Restricted Boltzmann Machines further restrict BMs

to those without visible-visible and hidden-hidden connections.

"""

from __future__ import print_function

import timeit

try:

import PIL.Image as Image

except ImportError:

import Image

import numpy

import theano

import theano.tensor as T

import os

from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams

from utils import tile_raster_images

from logistic_sgd import load_data

# start-snippet-1

class RBM(object):

"""Restricted Boltzmann Machine (RBM) """

def __init__(

self,

input=None,

n_visible=784,

n_hidden=500,

W=None,

hbias=None,

vbias=None,

numpy_rng=None,

theano_rng=None

):

"""

RBM constructor. Defines the parameters of the model along with

basic operations for inferring hidden from visible (and vice-versa),

as well as for performing CD updates.

:param input: None for standalone RBMs or symbolic variable if RBM is

part of a larger graph.

:param n_visible: number of visible units

:param n_hidden: number of hidden units

:param W: None for standalone RBMs or symbolic variable pointing to a

shared weight matrix in case RBM is part of a DBN network; in a DBN,

the weights are shared between RBMs and layers of a MLP

:param hbias: None for standalone RBMs or symbolic variable pointing

to a shared hidden units bias vector in case RBM is part of a

different network

:param vbias: None for standalone RBMs or a symbolic variable

pointing to a shared visible units bias

"""

self.n_visible = n_visible

self.n_hidden = n_hidden

if numpy_rng is None:

# create a number generator

numpy_rng = numpy.random.RandomState(1234)

if theano_rng is None:

theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))

if W is None:

# W is initialized with `initial_W` which is uniformely

# sampled from -4*sqrt(6./(n_visible+n_hidden)) and

# 4*sqrt(6./(n_hidden+n_visible)) the output of uniform if

# converted using asarray to dtype theano.config.floatX so

# that the code is runable on GPU

initial_W = numpy.asarray(

numpy_rng.uniform(

low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)),

high=4 * numpy.sqrt(6. / (n_hidden + n_visible)),

size=(n_visible, n_hidden)

),

dtype=theano.config.floatX

)

# theano shared variables for weights and biases

W = theano.shared(value=initial_W, name='W', borrow=True)

if hbias is None:

# create shared variable for hidden units bias

hbias = theano.shared(

value=numpy.zeros(

n_hidden,

dtype=theano.config.floatX

),

name='hbias',

borrow=True

)

if vbias is None:

# create shared variable for visible units bias

vbias = theano.shared(

value=numpy.zeros(

n_visible,

dtype=theano.config.floatX

),

name='vbias',

borrow=True

)

# initialize input layer for standalone RBM or layer0 of DBN

self.input = input

if not input:

self.input = T.matrix('input')

self.W = W

self.hbias = hbias

self.vbias = vbias

self.theano_rng = theano_rng

# **** WARNING: It is not a good idea to put things in this list

# other than shared variables created in this function.

self.params = [self.W, self.hbias, self.vbias]

# end-snippet-1

def free_energy(self, v_sample):

''' Function to compute the free energy '''

wx_b = T.dot(v_sample, self.W) + self.hbias

vbias_term = T.dot(v_sample, self.vbias)

hidden_term = T.sum(T.log(1 + T.exp(wx_b)), axis=1)

return -hidden_term - vbias_term

def propup(self, vis):

'''This function propagates the visible units activation upwards to

the hidden units

Note that we return also the pre-sigmoid activation of the

layer. As it will turn out later, due to how Theano deals with

optimizations, this symbolic variable will be needed to write

down a more stable computational graph (see details in the

reconstruction cost function)

'''

pre_sigmoid_activation = T.dot(vis, self.W) + self.hbias

return [pre_sigmoid_activation, T.nnet.sigmoid(pre_sigmoid_activation)]

def sample_h_given_v(self, v0_sample):

''' This function infers state of hidden units given visible units '''

# compute the activation of the hidden units given a sample of

# the visibles

pre_sigmoid_h1, h1_mean = self.propup(v0_sample)

# get a sample of the hiddens given their activation

# Note that theano_rng.binomial returns a symbolic sample of dtype

# int64 by default. If we want to keep our computations in floatX

# for the GPU we need to specify to return the dtype floatX

h1_sample = self.theano_rng.binomial(size=h1_mean.shape,

n=1, p=h1_mean,

dtype=theano.config.floatX)

return [pre_sigmoid_h1, h1_mean, h1_sample]

def propdown(self, hid):

'''This function propagates the hidden units activation downwards to

the visible units

Note that we return also the pre_sigmoid_activation of the

layer. As it will turn out later, due to how Theano deals with

optimizations, this symbolic variable will be needed to write

down a more stable computational graph (see details in the

reconstruction cost function)

'''

pre_sigmoid_activation = T.dot(hid, self.W.T) + self.vbias

return [pre_sigmoid_activation, T.nnet.sigmoid(pre_sigmoid_activation)]

def sample_v_given_h(self, h0_sample):

''' This function infers state of visible units given hidden units '''

# compute the activation of the visible given the hidden sample

pre_sigmoid_v1, v1_mean = self.propdown(h0_sample)

# get a sample of the visible given their activation

# Note that theano_rng.binomial returns a symbolic sample of dtype

# int64 by default. If we want to keep our computations in floatX

# for the GPU we need to specify to return the dtype floatX

v1_sample = self.theano_rng.binomial(size=v1_mean.shape,

n=1, p=v1_mean,

dtype=theano.config.floatX)

return [pre_sigmoid_v1, v1_mean, v1_sample]

def gibbs_hvh(self, h0_sample):

''' This function implements one step of Gibbs sampling,

starting from the hidden state'''

pre_sigmoid_v1, v1_mean, v1_sample = self.sample_v_given_h(h0_sample)

pre_sigmoid_h1, h1_mean, h1_sample = self.sample_h_given_v(v1_sample)

return [pre_sigmoid_v1, v1_mean, v1_sample,

pre_sigmoid_h1, h1_mean, h1_sample]

def gibbs_vhv(self, v0_sample):

''' This function implements one step of Gibbs sampling,

starting from the visible state'''

pre_sigmoid_h1, h1_mean, h1_sample = self.sample_h_given_v(v0_sample)

pre_sigmoid_v1, v1_mean, v1_sample = self.sample_v_given_h(h1_sample)

return [pre_sigmoid_h1, h1_mean, h1_sample,

pre_sigmoid_v1, v1_mean, v1_sample]

# start-snippet-2

def get_cost_updates(self, lr=0.1, persistent=None, k=1):

"""This functions implements one step of CD-k or PCD-k

:param lr: learning rate used to train the RBM

:param persistent: None for CD. For PCD, shared variable

containing old state of Gibbs chain. This must be a shared

variable of size (batch size, number of hidden units).

:param k: number of Gibbs steps to do in CD-k/PCD-k

Returns a proxy for the cost and the updates dictionary. The

dictionary contains the update rules for weights and biases but

also an update of the shared variable used to store the persistent

chain, if one is used.

"""

# compute positive phase

pre_sigmoid_ph, ph_mean, ph_sample = self.sample_h_given_v(self.input)

# decide how to initialize persistent chain:

# for CD, we use the newly generate hidden sample

# for PCD, we initialize from the old state of the chain

if persistent is None:

chain_start = ph_sample

else:

chain_start = persistent

# end-snippet-2

# perform actual negative phase

# in order to implement CD-k/PCD-k we need to scan over the

# function that implements one gibbs step k times.

# Read Theano tutorial on scan for more information :

# http://deeplearning.net/software/theano/library/scan.html

# the scan will return the entire Gibbs chain

(

[

pre_sigmoid_nvs,

nv_means,

nv_samples,

pre_sigmoid_nhs,

nh_means,

nh_samples

],

updates

) = theano.scan(

self.gibbs_hvh,

# the None are place holders, saying that

# chain_start is the initial state corresponding to the

# 6th output

outputs_info=[None, None, None, None, None, chain_start],

n_steps=k,

name="gibbs_hvh"

)

# start-snippet-3

# determine gradients on RBM parameters

# note that we only need the sample at the end of the chain

chain_end = nv_samples[-1]

cost = T.mean(self.free_energy(self.input)) - T.mean(

self.free_energy(chain_end))

# We must not compute the gradient through the gibbs sampling

gparams = T.grad(cost, self.params, consider_constant=[chain_end])

# end-snippet-3 start-snippet-4

# constructs the update dictionary

for gparam, param in zip(gparams, self.params):

# make sure that the learning rate is of the right dtype

updates[param] = param - gparam * T.cast(

lr,

dtype=theano.config.floatX

)

if persistent:

# Note that this works only if persistent is a shared variable

updates[persistent] = nh_samples[-1]

# pseudo-likelihood is a better proxy for PCD

monitoring_cost = self.get_pseudo_likelihood_cost(updates)

else:

# reconstruction cross-entropy is a better proxy for CD

monitoring_cost = self.get_reconstruction_cost(updates,

pre_sigmoid_nvs[-1])

return monitoring_cost, updates

# end-snippet-4

def get_pseudo_likelihood_cost(self, updates):

"""Stochastic approximation to the pseudo-likelihood"""

# index of bit i in expression p(x_i | x_{\i})

bit_i_idx = theano.shared(value=0, name='bit_i_idx')

# binarize the input image by rounding to nearest integer

xi = T.round(self.input)

# calculate free energy for the given bit configuration

fe_xi = self.free_energy(xi)

# flip bit x_i of matrix xi and preserve all other bits x_{\i}

# Equivalent to xi[:,bit_i_idx] = 1-xi[:, bit_i_idx], but assigns

# the result to xi_flip, instead of working in place on xi.

xi_flip = T.set_subtensor(xi[:, bit_i_idx], 1 - xi[:, bit_i_idx])

# calculate free energy with bit flipped

fe_xi_flip = self.free_energy(xi_flip)

# equivalent to e^(-FE(x_i)) / (e^(-FE(x_i)) + e^(-FE(x_{\i})))

cost = T.mean(self.n_visible * T.log(T.nnet.sigmoid(fe_xi_flip -

fe_xi)))

# increment bit_i_idx % number as part of updates

updates[bit_i_idx] = (bit_i_idx + 1) % self.n_visible

return cost

def get_reconstruction_cost(self, updates, pre_sigmoid_nv):

"""Approximation to the reconstruction error

Note that this function requires the pre-sigmoid activation as

input. To understand why this is so you need to understand a

bit about how Theano works. Whenever you compile a Theano

function, the computational graph that you pass as input gets

optimized for speed and stability. This is done by changing

several parts of the subgraphs with others. One such

optimization expresses terms of the form log(sigmoid(x)) in

terms of softplus. We need this optimization for the

cross-entropy since sigmoid of numbers larger than 30. (or

even less then that) turn to 1. and numbers smaller than

-30. turn to 0 which in terms will force theano to compute

log(0) and therefore we will get either -inf or NaN as

cost. If the value is expressed in terms of softplus we do not

get this undesirable behaviour. This optimization usually

works fine, but here we have a special case. The sigmoid is

applied inside the scan op, while the log is

outside. Therefore Theano will only see log(scan(..)) instead

of log(sigmoid(..)) and will not apply the wanted

optimization. We can not go and replace the sigmoid in scan

with something else also, because this only needs to be done

on the last step. Therefore the easiest and more efficient way

is to get also the pre-sigmoid activation as an output of

scan, and apply both the log and sigmoid outside scan such

that Theano can catch and optimize the expression.

"""

cross_entropy = T.mean(

T.sum(

self.input * T.log(T.nnet.sigmoid(pre_sigmoid_nv)) +

(1 - self.input) * T.log(1 - T.nnet.sigmoid(pre_sigmoid_nv)),

axis=1

)

)

return cross_entropy

def test_rbm(learning_rate=0.1, training_epochs=15,

dataset='mnist.pkl.gz', batch_size=20,

n_chains=20, n_samples=10, output_folder='rbm_plots',

n_hidden=500):

"""

Demonstrate how to train and afterwards sample from it using Theano.

This is demonstrated on MNIST.

:param learning_rate: learning rate used for training the RBM

:param training_epochs: number of epochs used for training

:param dataset: path the the pickled dataset

:param batch_size: size of a batch used to train the RBM

:param n_chains: number of parallel Gibbs chains to be used for sampling

:param n_samples: number of samples to plot for each chain

"""

datasets = load_data(dataset)

train_set_x, train_set_y = datasets[0]

test_set_x, test_set_y = datasets[2]

# compute number of minibatches for training, validation and testing

n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size

# allocate symbolic variables for the data

index = T.lscalar() # index to a [mini]batch

x = T.matrix('x') # the data is presented as rasterized images

rng = numpy.random.RandomState(123)

theano_rng = RandomStreams(rng.randint(2 ** 30))

# initialize storage for the persistent chain (state = hidden

# layer of chain)

persistent_chain = theano.shared(numpy.zeros((batch_size, n_hidden),

dtype=theano.config.floatX),

borrow=True)

# construct the RBM class

rbm = RBM(input=x, n_visible=28 * 28,

n_hidden=n_hidden, numpy_rng=rng, theano_rng=theano_rng)

# get the cost and the gradient corresponding to one step of CD-15

cost, updates = rbm.get_cost_updates(lr=learning_rate,

persistent=persistent_chain, k=15)

#################################

# Training the RBM #

#################################

if not os.path.isdir(output_folder):

os.makedirs(output_folder)

os.chdir(output_folder)

# start-snippet-5

# it is ok for a theano function to have no output

# the purpose of train_rbm is solely to update the RBM parameters

train_rbm = theano.function(

[index],

cost,

updates=updates,

givens={

x: train_set_x[index * batch_size: (index + 1) * batch_size]

},

name='train_rbm'

)

plotting_time = 0.

start_time = timeit.default_timer()

# go through training epochs

for epoch in range(training_epochs):

# go through the training set

mean_cost = []

for batch_index in range(n_train_batches):

mean_cost += [train_rbm(batch_index)]

print('Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost))

# Plot filters after each training epoch

plotting_start = timeit.default_timer()

# Construct image from the weight matrix

image = Image.fromarray(

tile_raster_images(

X=rbm.W.get_value(borrow=True).T,

img_shape=(28, 28),

tile_shape=(10, 10),

tile_spacing=(1, 1)

)

)

image.save('filters_at_epoch_%i.png' % epoch)

plotting_stop = timeit.default_timer()

plotting_time += (plotting_stop - plotting_start)

end_time = timeit.default_timer()

pretraining_time = (end_time - start_time) - plotting_time

print ('Training took %f minutes' % (pretraining_time / 60.))

# end-snippet-5 start-snippet-6

#################################

# Sampling from the RBM #

#################################

# find out the number of test samples

number_of_test_samples = test_set_x.get_value(borrow=True).shape[0]

# pick random test examples, with which to initialize the persistent chain

test_idx = rng.randint(number_of_test_samples - n_chains)

persistent_vis_chain = theano.shared(

numpy.asarray(

test_set_x.get_value(borrow=True)[test_idx:test_idx + n_chains],

dtype=theano.config.floatX

)

)

# end-snippet-6 start-snippet-7

plot_every = 1000

# define one step of Gibbs sampling (mf = mean-field) define a

# function that does `plot_every` steps before returning the

# sample for plotting

(

[

presig_hids,

hid_mfs,

hid_samples,

presig_vis,

vis_mfs,

vis_samples

],

updates

) = theano.scan(

rbm.gibbs_vhv,

outputs_info=[None, None, None, None, None, persistent_vis_chain],

n_steps=plot_every,

name="gibbs_vhv"

)

# add to updates the shared variable that takes care of our persistent

# chain :.

updates.update({persistent_vis_chain: vis_samples[-1]})

# construct the function that implements our persistent chain.

# we generate the "mean field" activations for plotting and the actual

# samples for reinitializing the state of our persistent chain

sample_fn = theano.function(

[],

[

vis_mfs[-1],

vis_samples[-1]

],

updates=updates,

name='sample_fn'

)

# create a space to store the image for plotting ( we need to leave

# room for the tile_spacing as well)

image_data = numpy.zeros(

(29 * n_samples + 1, 29 * n_chains - 1),

dtype='uint8'

)

for idx in range(n_samples):

# generate `plot_every` intermediate samples that we discard,

# because successive samples in the chain are too correlated

vis_mf, vis_sample = sample_fn()

print(' ... plotting sample %d' % idx)

image_data[29 * idx:29 * idx + 28, :] = tile_raster_images(

X=vis_mf,

img_shape=(28, 28),

tile_shape=(1, n_chains),

tile_spacing=(1, 1)

)

# construct image

image = Image.fromarray(image_data)

image.save('samples.png')

# end-snippet-7

os.chdir('../')

if __name__ == '__main__':

test_rbm()