-
-
Save Jeksonic/7d08d107aa02b2cb207a5571506d14e8 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from __future__ import print_function, division | |
| import numpy as np | |
| import tensorflow as tf | |
| import matplotlib.pyplot as plt | |
| # hyperparameters | |
| state_size = 4 # size of hidden layer of neurons | |
| seq_length = 15 # number of steps to unroll the RNN for | |
| batch_size = 5 # number of samples to learn simultaneously | |
| learn_rate = 3e-1 # learning rate | |
| num_epochs = 100 | |
| total_series_length = 50000 | |
| num_classes = 2 | |
| num_features = 1 | |
| echo_step = 3 | |
| def generateData(): | |
| x = np.array(np.random.choice(2, total_series_length, p=[0.5, 0.5])) | |
| y = np.roll(x, echo_step) | |
| y[0:echo_step] = 0 | |
| x = x.reshape((batch_size, num_features, -1)) | |
| y = y.reshape((batch_size, -1)) | |
| return (x, y) | |
| def plot(loss_list, predictions_series, batchX, batchY): | |
| plt.subplot(2, 3, 1) | |
| plt.cla() | |
| plt.plot(loss_list) | |
| for batch_series_idx in range(batch_size): | |
| one_hot_output_series = np.array(predictions_series)[:, batch_series_idx, :] | |
| single_output_series = np.array([(1 if out[0] < 0.5 else 0) for out in one_hot_output_series]) | |
| plt.subplot(2, 3, batch_series_idx + 2) | |
| plt.cla() | |
| plt.axis([0, seq_length, 0, 2]) | |
| left_offset = range(seq_length) | |
| plt.bar(left_offset, batchX[batch_series_idx, :], width=1, color="blue") | |
| plt.bar(left_offset, batchY[batch_series_idx, :] * 0.5, width=1, color="red") | |
| plt.bar(left_offset, single_output_series * 0.3, width=1, color="green") | |
| plt.draw() | |
| plt.pause(0.0001) | |
| def run_vanilla_basicRNN(inputs_series, init_state): | |
| W = tf.Variable(np.random.rand(state_size + num_features, state_size), dtype=tf.float32) | |
| b = tf.Variable(np.zeros((num_features, state_size)), dtype=tf.float32) | |
| current_state = init_state | |
| states_series = [] | |
| for current_input in inputs_series: | |
| input_and_state_concatenated = tf.concat([current_input, current_state], 1) # Increasing number of columns | |
| next_state = tf.tanh(tf.matmul(input_and_state_concatenated, W) + b) # Broadcasted addition | |
| states_series.append(next_state) | |
| current_state = next_state | |
| return states_series, current_state | |
| tf.reset_default_graph() | |
| batchX_placeholder = tf.placeholder(tf.float32, [batch_size, num_features, seq_length]) | |
| batchY_placeholder = tf.placeholder(tf.int32, [batch_size, seq_length]) | |
| init_state = tf.placeholder(tf.float32, [batch_size, state_size]) | |
| W = tf.Variable(np.random.rand(state_size, num_classes),dtype=tf.float32) | |
| b = tf.Variable(np.zeros((1,num_classes)), dtype=tf.float32) | |
| # Unpack columns | |
| inputs_series = tf.unstack(batchX_placeholder, axis=2) | |
| labels_series = tf.unstack(batchY_placeholder, axis=1) | |
| # Forward pass | |
| states_series, current_state = run_vanilla_basicRNN(inputs_series, init_state) | |
| logits_series = [tf.matmul(state, W) + b for state in states_series] #Broadcasted addition | |
| predictions_series = [tf.nn.softmax(logits) for logits in logits_series] | |
| losses = [tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels) for logits, labels in zip(logits_series,labels_series)] | |
| total_loss = tf.reduce_mean(losses) | |
| train_step = tf.train.AdagradOptimizer(learn_rate).minimize(total_loss) | |
| with tf.Session() as sess: | |
| sess.run(tf.initialize_all_variables()) | |
| plt.ion() | |
| plt.figure() | |
| plt.show() | |
| loss_list = [] | |
| num_batches = total_series_length//batch_size//seq_length | |
| for epoch_idx in range(num_epochs): | |
| x,y = generateData() | |
| _current_state = np.zeros((batch_size, state_size)) | |
| print("New data, epoch", epoch_idx) | |
| for batch_idx in range(num_batches): | |
| start_idx = batch_idx * seq_length | |
| end_idx = start_idx + seq_length | |
| batchX = x[:,:,start_idx:end_idx] | |
| batchY = y[:,start_idx:end_idx] | |
| _total_loss, _train_step, _current_state, _predictions_series = sess.run( | |
| [total_loss, train_step, current_state, predictions_series], | |
| feed_dict={ | |
| batchX_placeholder:batchX, | |
| batchY_placeholder:batchY, | |
| init_state:_current_state | |
| }) | |
| loss_list.append(_total_loss) | |
| if batch_idx%100 == 0: | |
| print("Step",batch_idx, "Loss", _total_loss) | |
| plot(loss_list, _predictions_series, batchX[:,0,:], batchY) # plot only first feature | |
| plt.ioff() | |
| plt.show() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment