Talk:DreamTeam
rbm.py 15 Feb 2017[edit source]
#!/usr/bin/env python3 import time import math import random from loader import MNIST print('Hello!') num_phase = 3 num_visible = 28 * 28 num_hidden = 28 * 28 temperature = 42 learning_rate = 0.00042 batch = 1 ABS_limit = num_phase mnist = MNIST() hidden = list(0 for i in range(num_hidden)) visible = list(0 for i in range(num_visible)) #2d weight matrix weight = list(list(random.gauss(0, 0.01) for j in range(num_hidden)) for i in range(num_visible)) weight_update = list(list(0 for j in range(num_hidden)) for i in range(num_visible)) def transfer_function(energy_val): """ try: res = 1 / (1 + math.e**(energy_val / temperature)) return res except OverflowError as e: print("for energy_val=[{e_val}]".format(e_val=energy_val), e) return 0 """ return 1 if energy_val >= 0 else 0 def update_hidden_layer(energy): for j, h_node in enumerate(hidden): hidden[j] = transfer_function(energy[j]) def update_visible_layer(energy): for i, v_node in enumerate(visible): visible[i] = transfer_function(energy[i]) def compute_hidden_energy(): energy = list(0 for j in range(num_hidden)) for j, h_node in enumerate(hidden): for i, v_node in enumerate(visible): energy[j] += v_node * weight[i][j] # print("Hidden Energy:", j, energy[j]) update_hidden_layer(energy) def compute_visible_energy(): energy = list(0 for i in range(num_visible)) for i, v_node in enumerate(visible): for j, h_node in enumerate(hidden): energy[i] += h_node * weight[i][j] # print("Visible Energy:", i, energy[i]) update_visible_layer(energy) def update_weights(mode): if mode is '+': for i, v_node in enumerate(visible): for j, h_node in enumerate(hidden): weight_update[i][j] += visible[i] * hidden[j] elif mode is '-': for i, v_node in enumerate(visible): for j, h_node in enumerate(hidden): weight_update[i][j] -= visible[i] * hidden[j] def alternate_gibbs(): for AGS_phase in range(1, num_phase + 1): if AGS_phase % 2: #is odd compute_hidden_energy() else: compute_visible_energy() if AGS_phase == 1: #? pseudo code says ABS_phase update_weights(mode='+') elif AGS_phase == ABS_limit: update_weights(mode='-') def apply_weight_update(): for i in range(num_visible): for j in range(num_hidden): weight[i][j] += learning_rate/batch * weight_update[i][j] def load_data(): m = MNIST() res = list(list() for i in range(10)) imgs, labels = m.load_training() for i, img in enumerate(imgs): for j, val in enumerate(img): img[j] = 1 if val > 127 else 0 res[labels[i]].append(img) return res # [[ZERO_0, ..., ZERO_n], ..., [NINE_0, ..., NINE_n]] def sample_batch(epoch): res = list() for i in range(10): res.append(random.choice(epoch[i])) random.shuffle(res) return res def still_learning(): return True train_data = load_data() while still_learning(): print("Starting minibatch") minibatch = sample_batch(train_data) for v in minibatch: visible = v print("%s" % mnist.display(visible, hidden)) alternate_gibbs() apply_weight_update() # print("Weights", weights) comment = """ for every batch : visible[] = get_datavector(batch) for every AGS_phase : if AGS_phase is odd : # Energy compute Eq.14 - 2 loops -> O(n^2) for every hidden_node : for every visible_node : energy[j] += visible[i]*weight[i][j] # Node select Eq.12 - 1 loop -> O(n) for every hidden_node : hidden[j] = transfer_function(energy[j]) else : # Energy compute Eq.13 - 2 loops -> O(n^2) for every visible_node : for every hidden_node : energy[i] += hidden[j]*weight[i][j] # Node select Eq.11 - 1 loop -> O(n) for every visible_node : visible[i] = transfer_function(energy[i]) # Weight update Eq.15 - 2 loops -> O(n^2) if ABS_phase == 1 : for every visible_node : for every hidden_node : weight_update[i][j] += visible[i] * hidden[j] else if ABS_phase == ABS_limit : for every visible_node : for every hidden_node : weight_update[i][j] -= visible[i] * hidden[j] # Weight update Eq.15 - 2 loops -> O(n^2) for every visible_node : for every hidden_node : weight[i][j] += learning_rate/batch * weight_update[i][ j] """ import os import struct from array import array class MNIST(object): def __init__(self, path='.'): self.path = path self.test_img_fname = 't10k-images-idx3-ubyte' self.test_lbl_fname = 't10k-labels-idx1-ubyte' self.train_img_fname = 'train-images-idx3-ubyte' self.train_lbl_fname = 'train-labels-idx1-ubyte' self.test_images = [] self.test_labels = [] self.train_images = [] self.train_labels = [] def load_testing(self): ims, labels = self.load(os.path.join(self.path, self.test_img_fname), os.path.join(self.path, self.test_lbl_fname)) self.test_images = ims self.test_labels = labels return ims, labels def load_training(self): ims, labels = self.load(os.path.join(self.path, self.train_img_fname), os.path.join(self.path, self.train_lbl_fname)) self.train_images = ims self.train_labels = labels return ims, labels @classmethod def load(cls, path_img, path_lbl): with open(path_lbl, 'rb') as file: magic, size = struct.unpack(">II", file.read(8)) if magic != 2049: raise ValueError('Magic number mismatch, expected 2049,' 'got {}'.format(magic)) labels = array("B", file.read()) with open(path_img, 'rb') as file: magic, size, rows, cols = struct.unpack(">IIII", file.read(16)) if magic != 2051: raise ValueError('Magic number mismatch, expected 2051,' 'got {}'.format(magic)) image_data = array("B", file.read()) images = [] for i in range(size): images.append([0] * rows * cols) for i in range(size): images[i][:] = image_data[i * rows * cols:(i + 1) * rows * cols] return images, labels @classmethod def display(cls, img0, img1, width=28, threshold=1): assert len(img0) == len(img1) render = '' img0s, img1s = '', '' for z, (i, j) in enumerate(zip(img0, img1)): if z % width == 0: render += img0s + ' ' + img1s + '\n' img0s, img1s = '', '' if i >= threshold: img0s += '@ ' else: img0s += '. ' if j >= threshold: img1s += '@ ' else: img1s += '. ' return render
Brainduino images[edit source]
Discussion 14 December 2016[edit source]
Restricted Boltzmann Machine
- is RBM not an RNN ?
- summary recollection - RBM is a type of NN.
Has layers. Input, output, hidden. Can have multiple hidden layers. Has nodes and weights.
What is a "Recurrent" Neural Network? Why not "Recursive"?
How is Error Backpropogation different concept from "Recurrent"?
Network architecture ... adjust weights within rather than redefine network topology
RNN may have generative capacity.
Recollections of current comprehension:
- it is restricted. ie connections not allowed within layer
RBM involves MCMC (Markov chain Monte Carlo) process via Gibbs Sampling
relates to Ising Model Hopfield nets
Discussion 07 October 2015[edit source]
Distributed neural networks
- hadoop - useful to handle reliability when dealing with thousands of servers
- YARN yet another resource negotiator
- apache - also mahout
- task oriented - wants to solve actual data analysis quickly - not energy efficient
- compare to BOINC or folding@home etc
- folding@home 40 petaflops - how might this support simulation of something like 10**11 neurons
- compare to spiNNaker - specialized network for thousands of ARM cores, designed for spiking neural networks but can support traditional backpropagation
Hacking Hardware[edit source]
Presentation @ Black Hat 2011, discusses JTAG https://media.blackhat.com/bh-dc-11/Grand/BlackHat_DC_2011_Grand-Workshop.pdf
Restricted Boltzmann Machine[edit source]
- roots in Hopfield network
- relevance of pruning - restricted connectivity makes training feasible for non-trivial number of neurons
https://github.com/search?q=neurosky&type=&ref=simplesearch (found 48 repository results)
https://github.com/nbdt == NB dream team
// Audio Spectrum Display // Copyright 2013 Tony DiCola (tony@tonydicola.com) // This code is part of the guide at http://learn.adafruit.com/fft-fun-with-fourier-transforms/ #define ARM_MATH_CM4 #include <arm_math.h> #include "FastLED.h" // ebw code #include <avr/pgmspace.h> // PROGMEM #include <Cycler.h> #include <EightBitWaves.h> //#define LED_COUNT 13 //#define LED_CLASS WS2812 //#define LED_COLOR_ORDER GRB #define LED_MAX_BRIGHTNESS 64 // 1/8 //#define LED_DT 2 // SERIAL DATA PIN //#define LED_CK 6 // SERIAL CLOCK PIN //#define SERIAL_BAUDRATE 9600 #define SERIAL_TIMEOUT 5 //Ticker ticker; //Cycler hue_cycler, brightness_cycler; Cycler brightness_cycler; //struct CRGB pixel, fastled_buffer[LED_COUNT]; uint8_t minWave = 0, maxWave = 255; uint32_t ticks_per_second = 1000; uint32_t prevMillis = millis(); // ebw code //////////////////////////////////////////////////////////////////////////////// // CONIFIGURATION // These values can be changed to alter the behavior of the spectrum display. //////////////////////////////////////////////////////////////////////////////// #define DEBUG false #define OUTPUT_LED_DATA false #define FULL_SET false #define BAUD_RATE 38400 #define NUM_LEDS 12 #define DATA_PIN 2 #define ARRAY_SZ(x) (sizeof(x) / sizeof((x)[0])) uint SAMPLE_RATE_HZ = 9000; // Sample rate of the audio in hertz. float SPECTRUM_MIN_DB = 30.0; // Audio intensity (in decibels) that maps to low LED brightness. float SPECTRUM_MAX_DB = 60.0; // Audio intensity (in decibels) that maps to high LED brightness. const int FFT_SIZE = 256; // Size of the FFT. Realistically can only be at most 256 // without running out of memory for buffers and other state. const int AUDIO_INPUT_PIN = 14; // Input ADC pin for audio data. const int ANALOG_READ_RESOLUTION = 10; // Bits of resolution for the ADC. const int ANALOG_READ_AVERAGING = 16; // Number of samples to average with each ADC reading. const int ONBOARD_LED_PIN = 13; // Output pin for power LED (pin 13 to use Teensy 3.0's onboard LED). const int MAX_CHARS = 65; // Max size of the input command buffer uint8_t brightness = 128; CRGB leds[NUM_LEDS]; //////////////////////////////////////////////////////////////////////////////// // INTERNAL STATE // These shouldn't be modified unless you know what you're doing. //////////////////////////////////////////////////////////////////////////////// IntervalTimer samplingTimer; float samples[FFT_SIZE*2]; float magnitudes[FFT_SIZE]; int sampleCounter = 0; char commandBuffer[MAX_CHARS]; float frequencyWindow[NUM_LEDS+1]; boolean ledState = false; uint loopCounter; //////////////////////////////////////////////////////////////////////////////// // setup //////////////////////////////////////////////////////////////////////////////// void setup() { // Set up serial port. Serial.begin(BAUD_RATE); delay(1000); loopCounter = 0; // Set up ADC and audio input. pinMode(AUDIO_INPUT_PIN, INPUT); analogReadResolution(ANALOG_READ_RESOLUTION); analogReadAveraging(ANALOG_READ_AVERAGING); // Turn on the power indicator LED. pinMode(ONBOARD_LED_PIN, OUTPUT); digitalWrite(ONBOARD_LED_PIN, LOW); FastLED.addLeds<NEOPIXEL, DATA_PIN>(leds, NUM_LEDS); // Clear the input command buffer memset(commandBuffer, 0, sizeof(commandBuffer)); // Initialize spectrum display spectrumSetup(); // Serial.println("** setup() spectrumSetup **********************"); // delay(1000); brightness_cycler.setup((float) 6.0, ticks_per_second); // Begin sampling audio samplingBegin(); } //////////////////////////////////////////////////////////////////////////////// // loop //////////////////////////////////////////////////////////////////////////////// void loop() { // Calculate FFT if a full sample is available. if (samplingIsDone()) { processFFT(); } updateBrightness(); } //////////////////////////////////////////////////////////////////////////////// // UTILITY FUNCTIONS //////////////////////////////////////////////////////////////////////////////// void updateBrightness() { uint16_t delta_ticks = (uint16_t)(millis() - prevMillis); prevMillis += delta_ticks; brightness_cycler.update(delta_ticks); // unsigned char brightness = EightBitWaves::sine(brightness_cycler.phase(), minWave, maxWave); brightness = EightBitWaves::sine(brightness_cycler.phase(), minWave, maxWave); } void processFFT() { // Run FFT on sample data. arm_cfft_radix4_instance_f32 fft_inst; // These functions have been depricated arm_cfft_radix4_init_f32(&fft_inst, FFT_SIZE, 0, 1); arm_cfft_radix4_f32(&fft_inst, samples); // arm_cfft_f32(&fft_inst, samples, 0, 1); // Calculate magnitude of complex numbers output by the FFT. arm_cmplx_mag_f32(samples, magnitudes, FFT_SIZE); updateLEDs(); // Restart audio sampling. samplingBegin(); } void outputFFTData() { if (DEBUG && FULL_SET) { Serial.println("** FFT Data ****************************"); Serial.print("** Total size of magnitude array: "); Serial.println(sizeof(magnitudes)); Serial.print("** Number of elements in magnitude: "); Serial.println(sizeof(magnitudes)/sizeof(*magnitudes)); // Serial.print("** Number of elements in magnitude: "); // Serial.println(sizeOfFloatArray(magnitudes)); } // for (int i = 0; i < sizeof(magnitudes)/sizeof(*magnitudes); ++i) { // was FFT_SIZE for (size_t i = 0; i < ARRAY_SZ(magnitudes); ++i) { float intensity = magnitudes[i]; intensity = 20.0*log10(intensity); // Scale the intensity and clamp between 0 and 1.0. intensity -= SPECTRUM_MIN_DB; intensity = intensity < 0.0 ? 0.0 : intensity; intensity /= (SPECTRUM_MAX_DB-SPECTRUM_MIN_DB); intensity = intensity > 1.0 ? 1.0 : intensity; if (FULL_SET) { if (DEBUG) { Serial.print("FFT["); Serial.print(i); Serial.print("] = "); Serial.println((byte)(254 * intensity)); } else { Serial.write((byte)(254 * intensity)); } } } if (FULL_SET) { if (DEBUG) { Serial.println("** End FFT Results **********************"); } else { Serial.write(255); } } } /* if (DEBUG) { Serial.print("FFT["); Serial.print(i); Serial.print("] = "); Serial.println(magnitudes[i]); } else { // Serial.write(); // send out the data Serial.write(255); // Send stop Byte } } */ //************************************************************************** // updateGrnLED //************************************************************************** // void updateGrnLED() { // The measured time between two consecutive events (rock solid on the scope): // FHT_N mSec Baud Rate // // Serial.print("updateOnboardLED"); digitalWrite(ONBOARD_LED_PIN, HIGH && ledState); // turn the LED on or off (HIGH is the voltage level) ledState = !ledState; } //************************************************************************** // frequencyToBin // Function - Convert a frequency to the appropriate FFT bin it will fall within. //************************************************************************** // int frequencyToBin(float frequency) { float binFrequency = float(SAMPLE_RATE_HZ) / float(FFT_SIZE); return int(frequency / binFrequency); } //************************************************************************** // updateLEDs // Function - Processes the FFT data and outputs to LEDs. Update each LED based // on the intensity of the audio in the associated frequency window. //************************************************************************** // void updateLEDs() { float intensity, otherMean; if (OUTPUT_LED_DATA && DEBUG) Serial.println("** Start LED FFT Results **********************"); // Serial.print("ARRAY_SZ(leds) = "); // Serial.print(ARRAY_SZ(leds)); // Serial.print(", sizeof magnitudes = "); // Serial.println(ARRAY_SZ(magnitudes)); for (size_t i = 0; i < ARRAY_SZ(leds); ++i) { // windowMean(magnitudes, // frequencyToBin(frequencyWindow[i]), windowMean(frequencyToBin(frequencyWindow[i]), frequencyToBin(frequencyWindow[i+1]), &intensity, &otherMean); // Serial.print("**1 intensity["); // Serial.print(i); // Serial.print("] = "); // Serial.println(intensity); // Convert intensity to decibels. intensity = 20.0*log10(intensity); // Scale the intensity and clamp between 0 and 1.0. intensity -= SPECTRUM_MIN_DB; intensity = intensity < 0.0 ? 0.0 : intensity; intensity /= (SPECTRUM_MAX_DB-SPECTRUM_MIN_DB); intensity = intensity > 1.0 ? 1.0 : intensity; // Serial.print("**2 intensity["); // Serial.print(i); // Serial.print("] = "); // Serial.println(intensity); // Output intensity value to LED float newHue = 255*intensity; if (newHue < 64) { newHue = 0; } else { newHue = map(newHue, 64, 255, 0, 255); } leds[i] = CHSV(newHue, 255, brightness); if (OUTPUT_LED_DATA) { if (DEBUG) { Serial.print("FFT["); Serial.print(i); Serial.print("] = "); Serial.println(255.0 * intensity); } else { // Serial.write((byte)(254 * intensity)); Serial.write((byte)(newHue)); } } } if (OUTPUT_LED_DATA) { if (DEBUG) { Serial.println("** End FFT Results **********************"); } else { Serial.write(255); } } // pixels.show(); FastLED.show(); } /* //************************************************************************** // setHSVColor // Function - Converts HSV values to RGB and assigns the result to the specified LED //************************************************************************** // void setHSVColor(int ledIndex, float hue, float saturation, float brightness) { uint32_t myColor = pixelHSVtoRGBColor(hue, saturation, brightness); pixels.setPixelColor(ledIndex, myColor); // Serial.print("LED num = "); // Serial.print(ledIndex); // Serial.print(", color = "); // Serial.println(myColor); } //************************************************************************** // setHSVColor // Function - Convert from HSV values to RGB colors usable by neo pixel functions. // hue: 0.0 - 360.0 // saturation: 0.0 - 1.0 // value: 0.0 - 1.0 //************************************************************************** // uint32_t pixelHSVtoRGBColor(float hue, float saturation, float value) { // Implemented from algorithm at http://en.wikipedia.org/wiki/HSL_and_HSV#From_HSV float chroma = value * saturation; float h1 = float(hue)/60.0; float x = chroma*(1.0-fabs(fmod(h1, 2.0)-1.0)); float r = 0; float g = 0; float b = 0; if (h1 < 1.0) { r = chroma; g = x; } else if (h1 < 2.0) { r = x; g = chroma; } else if (h1 < 3.0) { g = chroma; b = x; } else if (h1 < 4.0) { g = x; b = chroma; } else if (h1 < 5.0) { r = x; b = chroma; } else // h1 <= 6.0 { r = chroma; b = x; } float m = value - chroma; r += m; g += m; b += m; return pixels.Color(int(255*r), int(255*g), int(255*b)); } */ //////////////////////////////////////////////////////////////////////////////// // SPECTRUM DISPLAY FUNCTIONS /////////////////////////////////////////////////////////////////////////////// void spectrumSetup() { // Set the frequency window values by evenly dividing the possible frequency // spectrum across the number of neo pixels. // Serial.println("** spectrumSetup **********************"); float windowSize = (SAMPLE_RATE_HZ / 2.0) / float(NUM_LEDS); for (size_t i = 0; i < ARRAY_SZ(leds) + 1; ++i) { frequencyWindow[i] = i*windowSize; // Serial.print("frequencyWindow["); // Serial.print(i); // Serial.print("] = "); // Serial.println(frequencyWindow[i]); } } //************************************************************************** // windowMean // Function - Compute the average magnitude of a target frequency window vs. // all other frequencies. // Parameters magnitudes, // frequencyToBin(frequencyWindow[i]), // frequencyToBin(frequencyWindow[i+1]), // &intensity, // &otherMean //************************************************************************** // //void windowMean(float* magnitudes, uint lowBin, uint highBin, float* windowMean, float* otherMean) { void windowMean(uint lowBin, uint highBin, float* windowMean, float* otherMean) { *windowMean = 0; *otherMean = 0; // Notice the first magnitude bin is skipped because it represents the // average power of the signal. // Serial.print("sizeof magnitudes: "); // Serial.print( sizeof(magnitudes) ); // Serial.print(", ARRAY_SZ(magnitudes): "); // Serial.println( ARRAY_SZ(magnitudes) ); for (size_t i = 1; i < (ARRAY_SZ(magnitudes))/2; ++i) { // Serial.print("i = "); // Serial.print(i); // // Serial.print(", lowBin = "); // Serial.print(lowBin); // // Serial.print(", highBin = "); // Serial.print(highBin); if (i >= lowBin && i <= highBin) { *windowMean += magnitudes[i]; // Serial.print(", magnitude: "); // Serial.print(magnitudes[i]); } else { *otherMean += magnitudes[i]; } // Serial.println(); } *windowMean /= (highBin - lowBin) + 1; // Serial.print(" **** windowMean: "); // Serial.println(*windowMean); *otherMean /= (FFT_SIZE / 2 - (highBin - lowBin)); } //////////////////////////////////////////////////////////////////////////////// // SAMPLING FUNCTIONS //////////////////////////////////////////////////////////////////////////////// //************************************************************************** // samplingCallback // Function - Does an analog to digital conversion of the microphone input. //************************************************************************** // void samplingCallback() { // Read from the microphone input pin and store the sample data samples[sampleCounter] = (float32_t)analogRead(AUDIO_INPUT_PIN); // Complex FFT functions require a coefficient for the imaginary part of the input. // Since we only have real data, set this coefficient to zero. samples[sampleCounter+1] = 0.0; // Update sample buffer position and stop after the buffer is filled sampleCounter += 2; if (sampleCounter >= FFT_SIZE*2) { samplingTimer.end(); } } //************************************************************************** // samplingBegin // Function - Starts the microphone ADC sampling timer function. //************************************************************************** // void samplingBegin() { // Reset sample buffer position and start callback at necessary rate. sampleCounter = 0; samplingTimer.begin(samplingCallback, 1000000/SAMPLE_RATE_HZ); // 1,000,000 / 9000 = 111 uSec } //************************************************************************** // samplingIsDone // Function - Indicates if a complete sample set of microphone data has been captured. //************************************************************************** // boolean samplingIsDone() { return sampleCounter >= FFT_SIZE*2; } //////////////////////////////////////////////////////////////////////////////// // COMMAND PARSING FUNCTIONS // These functions allow parsing simple commands input on the serial port. // Commands allow reading and writing variables that control the device. // // All commands must end with a semicolon character. // // Example commands are: // GET SAMPLE_RATE_HZ; // - Get the sample rate of the device. // SET SAMPLE_RATE_HZ 400; // - Set the sample rate of the device to 400 hertz. // //////////////////////////////////////////////////////////////////////////////// void parserLoop() { // Process any incoming characters from the serial port while (Serial.available() > 0) { char c = Serial.read(); // Add any characters that aren't the end of a command (semicolon) to the input buffer. if (c != ';') { c = toupper(c); strncat(commandBuffer, &c, 1); } else { // Parse the command because an end of command token was encountered. parseCommand(commandBuffer); // Clear the input buffer memset(commandBuffer, 0, sizeof(commandBuffer)); } } } // Macro used in parseCommand function to simplify parsing get and set commands for a variable #define GET_AND_SET(variableName) \ else if (strcmp(command, "GET " #variableName) == 0) { \ Serial.println(variableName); \ } \ else if (strstr(command, "SET " #variableName " ") != NULL) { \ variableName = (typeof(variableName)) atof(command+(sizeof("SET " #variableName " ")-1)); \ } //************************************************************************** // parseCommand // Function - Parses commands received via the serial input. //************************************************************************** // void parseCommand(char* command) { if (strcmp(command, "GET MAGNITUDES") == 0) { for (size_t i = 0; i < ARRAY_SZ(magnitudes); ++i) { Serial.println(magnitudes[i]); } } else if (strcmp(command, "GET SAMPLES") == 0) { for (size_t i = 0; i < ARRAY_SZ(samples); i+=2) { Serial.println(samples[i]); } } else if (strcmp(command, "GET FFT_SIZE") == 0) { Serial.println(FFT_SIZE); } GET_AND_SET(SAMPLE_RATE_HZ) // GET_AND_SET(LEDS_ENABLED) GET_AND_SET(SPECTRUM_MIN_DB) GET_AND_SET(SPECTRUM_MAX_DB) // Update spectrum display values if sample rate was changed. if (strstr(command, "SET SAMPLE_RATE_HZ ") != NULL) { spectrumSetup(); } }