main.cpp

00001 /* BlinkTalk - Bob Stone June 2013
00002  * Hands-free control of a speech synthesis system by blinking.
00003  *
00004  * This project implements a proof of concept speech system actuated by a single input
00005  * - this could be a button, but for a more fun application I am going to use blinks as
00006  * detected by an EEG headset, with a row & column time-sweep keyboard, inspired by
00007  * (but not the same as) the speech system used by Prof Stephen Hawking, mainly because the
00008  * Emic2 voice board includes the DECTalk 'PerfectPaul' voice familiar to all as Hawking.
00009  *
00010  * In Hawking's actual system, an infrared sensor mounted on his glasses detects a movement
00011  * of his cheek muscle, selecting letters from a keyboard  sweeping rows and columns, with a
00012  * predictive text system handling word completion and suggested follow-on words.  In
00013  * our version we will detect a blink using a Neurosky Mindwave Mobile headset and use it to
00014  * stop and select a cursor sweeping a keyboard grid.
00015  *
00016  * Hardware:
00017  *      Neurosky Mindwave Mobile headset - sends serial data packets over BlueTooth
00018  *      BlueSMiRF Silver Mate - receives data over BlueTooth from headset and relays over serial to mbed
00019  *      Parallax / Grand Idea Studio Emic2 speech synthesis - speaks text sent over serial from the mbed
00020  *      MikroElektronika TFT Proto - 320x240 TFT display with HX8374 controller driven by SPI
00021  *      Sparkfun level shifter - translates Emic2 5V serial to 3.3V serial for mbed
00022  *
00023  * Connections:
00024  *  mbed        BlueSMiRF   LevelShift  Emic2   Speaker TFT-PROTO
00025  *   GND         GND         GND/GND     GND             GND
00026  *   VOUT(3.3V)  VCC         LV                          3.3V
00027  *   VU (5V)                 HV          5V
00028  *   p9 (TX)     RX-I
00029  *   p10(RX)     TX-O
00030  *   p11(MOSI)                                           SDI
00031  *   p12(MISO)                                           SDO
00032  *   p13(SCLK)                                           SCL
00033  *   p14                                                 CS
00034  *   p15                                                 RST
00035  *   p27(RX)                 LV:RXO
00036  *   p28(TX)                 LV:TXI
00037  *                           HV:RXI      SOUT
00038  *                           HV:TXO      SN
00039  *                                       SP+     +
00040  *                                       SP-     -
00041  *      
00042  * Borrows from http://developer.neurosky.com/docs/doku.php?id=mindwave_mobile_and_arduino
00043  * Display library from Peter Drescher: http://mbed.org/cookbook/SPI-driven-QVGA-TFT
00044  */
00045 #include "mbed.h"
00046 #include "SPI_TFT.h"
00047 #include "Arial12x12.h"
00048 #include "Arial28x28.h"
00049 #include <string>
00050 
00051 //Peripherals
00052 Serial blueSmirf(p9, p10);                      //bluetooth comms (TX, RX)
00053 Serial voice(p28,p27);                          //Emic2 text to speech voice synth
00054 SPI_TFT screen(p11, p12, p13, p14, p15, "TFT"); //display
00055 
00056 //control variables
00057 int quality=0;
00058 bool connected=false;
00059 bool started=false;
00060 int row=-1;
00061 int col=-1;
00062 Timer initialDelay;
00063 
00064 //Keyboard - rows to display
00065 #define ROWS 5
00066 #define COLS 10
00067 //keys or text to display
00068 string keys[ROWS][COLS] = {
00069     {"1","2","3","4","5","6","7","8","9","0"},
00070     {"Q","W","E","R","T","Y","U","I","O","P"},
00071     {"A","S","D","F","G","H","J","K","L","!"},
00072     {"Z","X","C","V","B","N","M",",",".","?"},
00073     {"Space","\"SAY!\"","Delete"}
00074 };
00075 //positions to draw a line at the end of a column
00076 int keysColPos[ROWS][COLS] = {
00077     {36,67,98,129,160,191,222,253,284,315},
00078     {36,67,98,129,160,191,222,253,284,315},
00079     {36,67,98,129,160,191,222,253,284,315},
00080     {36,67,98,129,160,191,222,253,284,315},
00081     {100,210,315}
00082 };
00083 //number of elements in each row
00084 int keysRowSize[ROWS] = {10, 10, 10, 10, 3};
00085 
00086 //text being typed
00087 string typingBuffer("");
00088 
00089 //*****************************
00090 //User routines to process data
00091 //*****************************
00092 
00093 /** Say some text out loud
00094  *
00095  * @param text The text to say out loud
00096  */
00097 void say(string text)
00098 {
00099     voice.printf("S%s\n", text);//send command to speak to the Emic2
00100     while(!voice.readable())    //wait for Emic2 to return ':'
00101         ;                       //do nothing whilst it's still speaking
00102     voice.getc();               //pop the ':' response from the stream
00103 }
00104 
00105 /** Maps a value from one scale to another
00106  *
00107  * @param value Value we're trying to scale
00108  * @param min,max The range that value came from
00109  * @param newMin,newMax The new range we're scaling value into
00110  * @returns value mapped into new scale
00111  */
00112 int map(int value, int min, int max, int newMin, int newMax)
00113 {
00114     if (min==max)
00115         return newMax;
00116     else
00117         return newMin + (newMax-newMin) * (value-min) / (max-min);
00118 }
00119 
00120 /** Returns a 16-bit RGB565 colour from three 8-bit component values.
00121  *
00122  * @param red,green,blue primary colour channel values expressed as 0-255 each
00123  * @returns 16-bit RGB565 colour constructed as RRRRRGGGGGGBBBBB
00124  */
00125 int RGBColour(int red, int green, int blue)
00126 {
00127     //take most-significant parts of red, green and blue and bit-shift into RGB565 positions
00128     return ((red & 0xf8) << 8) | ((green & 0xfc) << 3) | ((blue & 0xf8) >> 3);
00129 }
00130 
00131 /** Returns a colour mapped on a gradient from one colour to another.
00132  *
00133  * @param value Value we're trying to pick a colour for
00134  * @param min,max Scale that value belongs in
00135  * @param minColour,maxColour start and end colours of the gradient we're choosing from (16-bit RGB565)
00136  * @returns colour that's as far along the gradient from minColour to maxColour as value is between min and max (16-bit RGB565)
00137  */
00138 int getMappedColour(int value, int min, int max, int minColour, int maxColour)
00139 {
00140     // TFT screen colours are 16-bit RGB565 i.e. RRRRRGGGGGGBBBBB
00141     int minRed = (minColour & 0xf800) >> 11; //bitmask for 5 bits red
00142     int maxRed = (maxColour & 0xf800) >> 11;
00143     int minGreen = (minColour & 0x7e0) >> 5; //bitmask for 6 bits green
00144     int maxGreen = (maxColour & 0x7e0) >> 5;
00145     int minBlue = minColour & 0x1f; // bitmask for 5 bits blue
00146     int maxBlue = maxColour & 0x1f;
00147     int valRed = map(value, min, max, minRed, maxRed);
00148     int valGreen = map(value, min, max, minGreen, maxGreen);
00149     int valBlue = map(value, min, max, minBlue, maxBlue);
00150     int valColour = ((valRed & 0x1F) << 11) | ((valGreen & 0x3F) << 5) | (valBlue & 0x1F);
00151     return valColour;
00152 }
00153 
00154 /** Displays a bar graph showing 'value' on a scale 'min' to 'max', where coords (x0,y0) are at 'min' and (x1,y1) are at 'max'.
00155  *
00156  * @param x0,y0 coordinates of the 'min' end of the bargraph
00157  * @param x1,y1 coordinates of the 'max' end of the bargraph
00158  * @param isHorizontal If true, bar graph will be drawn with horizontal bars
00159  * @param value Value of the bar, with bars drawn from min up to value, remaining 'backColour' from there to max
00160  * @param min,max Scale of the bar graph that value should be found within
00161  * @param minColour,maxColour colours at the min and max ends of the bar, drawn in a gradient between the two (16-bit RGB565)
00162  * @param backColour background colour of the bar graph (16-bit RGB565)
00163  */
00164 void displayBarGraph(int x0, int y0, int x1, int y1, bool isHorizontal, int value, int min, int max, int minColour, int maxColour, int backColour)
00165 {
00166     int valColour;
00167     if (isHorizontal) {
00168         if (x1>x0) {
00169             for (int i = x0; i < x1; i+=5) {
00170                 if (map(i, x0, x1, min, max) > value)
00171                     valColour = backColour;
00172                 else
00173                     valColour = getMappedColour(i, x0, x1, minColour, maxColour);
00174                 screen.fillrect(i, y0, i+3, y1, valColour);
00175             }
00176         } else {
00177             for (int i = x1; i < x0; i+=5) {
00178                 if (map(i, x0, x1, min, max) > value)
00179                     valColour = backColour;
00180                 else
00181                     valColour = getMappedColour(i, x0, x1, minColour, maxColour);
00182                 screen.fillrect(i-3, y0, i, y1, valColour);
00183             }
00184         }
00185     } else {
00186         if (y1>y0) {
00187             for (int i = y0; i < y1; i+=5) {
00188                 if (map(i, y0, y1, min, max) > value)
00189                     valColour = backColour;
00190                 else
00191                     valColour = getMappedColour(i, y0, y1, minColour, maxColour);
00192                 screen.fillrect(x0, i, x1, i+3, valColour);
00193             }
00194         } else {
00195             for (int i = y1; i < y0; i+=5) {
00196                 if (map(i, y0, y1, min, max) > value)
00197                     valColour = backColour;
00198                 else
00199                     valColour = getMappedColour(i, y0, y1, minColour, maxColour);
00200                 screen.fillrect(x0, i-3, x1, i, valColour);
00201             }
00202         }
00203     }
00204 }
00205 
00206 /** Draw a keyboard based on the supplied array of cells
00207  *
00208  * @param cells the cells to draw
00209  * @param rowHighlight off if -1, else highlights the numbered row
00210  * @param colHighlight off if -1, else highlights the cell in row/col
00211  */
00212 void drawKeyboard(int rowSize[ROWS], string cells[ROWS][COLS], int colPos[ROWS][COLS], int rowHighlight, int colHighlight)
00213 {
00214     int lineColour = RGBColour(0x20,0xFF,0xE0);
00215     screen.foreground(RGBColour(0xE0,0xC0,0x10));
00216     screen.set_font((unsigned char*) Arial28x28);
00217     for (int i=0; i<ROWS; i++) {
00218         int yPos=111+i*32;
00219         for (int j=0; j< rowSize[i]; j++) {
00220             if (j>0)
00221                 screen.locate(colPos[i][j-1]+5,84+i*32);
00222             else
00223                 screen.locate(10,84+i*32);
00224             screen.printf("%s",cells[i][j]);
00225             screen.line(colPos[i][j],yPos-31,colPos[i][j],yPos, lineColour);
00226         }
00227         screen.line(5, yPos, 315, yPos, lineColour);
00228     }
00229     screen.line(5, 79, 315, 79, lineColour);
00230     screen.line(5, 79, 5, 239, lineColour);
00231     if (rowHighlight >= 0) {
00232         if (colHighlight >= 0) { //highlight a cell
00233             if (colHighlight==0)
00234                 screen.rect(5,79+rowHighlight*32,colPos[rowHighlight][0],111+rowHighlight*32,RGBColour(0xFF,0x40,0x40));
00235             else
00236                 screen.rect(colPos[rowHighlight][colHighlight-1],79+rowHighlight*32,colPos[rowHighlight][colHighlight],111+rowHighlight*32,RGBColour(0xFF,0x40,0x40));
00237         } else { // highlight whole row
00238             screen.rect(5,79+rowHighlight*32,315,111+rowHighlight*32,RGBColour(0xFF,0x40,0x40));
00239         }
00240     }
00241 }
00242 
00243 void drawText()
00244 {
00245     screen.rect(5,35,315,74,RGBColour(0x20,0xFF,0xE0));
00246     screen.locate(10,40);
00247     screen.foreground(RGBColour(0xE0,0xC0,0x10));
00248     screen.set_font((unsigned char*) Arial28x28);
00249     screen.printf("%s_  ", typingBuffer);
00250 }
00251 
00252 /** This will be called if you blink.
00253  */
00254 void blinked(void)
00255 {
00256     //draw blink indicator
00257     if (quality == 0) {
00258         screen.fillrect(313, 13, 317, 17, White);
00259     }
00260     //select row or cell
00261     if (col == -1)
00262         col=-2;
00263     else {
00264         string s = keys[row][col];
00265         if (s.compare("Space") == 0)
00266             s=" ";
00267         if (s.compare("Delete") == 0) {
00268             s="";
00269             if (typingBuffer.length() > 0) {
00270                 typingBuffer = typingBuffer.substr(0, typingBuffer.length() -1);
00271             }
00272         } else if (s.compare("\"SAY!\"") == 0) {
00273             say(typingBuffer);
00274             screen.fillrect(5,35,315,74,Black);
00275             typingBuffer="";
00276         } else {
00277             typingBuffer.append(s);
00278         }
00279         row = -1;
00280         col = -1;
00281         drawText();
00282     }
00283 }
00284 
00285 /** This will be called when processed eSense data comes in, about once a second.
00286  *
00287  * @param poorQuality will be 0 if connections are good, 200 if connections are useless, and somewhere in between if connection dodgy.
00288  * @param attention processed percentage denoting focus and attention.  0 to 100
00289  * @param meditation processed percentage denoting calmness and serenity.  0 to 100
00290  * @param timeSinceLastPacket time since last packet processed, in milliseconds.
00291  */
00292 void eSenseData(int poorQuality, int attention, int meditation, int timeSinceLastPacket)
00293 {
00294     //quality indicator
00295     quality=poorQuality;
00296     if (poorQuality == 200)
00297         screen.fillrect(313, 3, 317, 7, Red);
00298     else if (poorQuality == 0)
00299         screen.fillrect(313, 3, 317, 7, Green);
00300     else
00301         screen.fillrect(313, 3, 317, 7, Yellow);
00302 
00303     //minimal eSense bars up at the top of the screen
00304     screen.set_font((unsigned char*) Arial12x12);
00305     if (attention > 0) {
00306         displayBarGraph(200, 5, 310, 15, true, attention, 0, 100, RGBColour(0x10,0x00,0x00), RGBColour(0xFF,0x00,0x00), 0x00);
00307         screen.locate(135, 6);
00308         screen.foreground(Red);
00309         screen.printf("Att: %d ",attention);
00310     }
00311     if (meditation > 0) {
00312         displayBarGraph(200, 18, 310, 28, true, meditation, 0, 100, RGBColour(0x00,0x10,0x00), RGBColour(0x00,0xFF,0x00), 0x00);
00313         screen.locate(128, 19);
00314         screen.foreground(Green);
00315         screen.printf("Med: %d ",meditation);
00316     }
00317     //clear blink indicator
00318     screen.fillrect(313, 13, 317, 17, Black);
00319     //Safe to start yet?
00320     if (initialDelay.read() == 0)
00321         initialDelay.start();
00322     else if (initialDelay.read() > 5) {
00323         started=true;
00324         initialDelay.stop();
00325     }
00326 }
00327 
00328 /** This will be called when processed meter reading data arrives, about once a second.
00329  * This is a breakdown of frequencies in the wave data into 8 named bands, these are:
00330  *   0: Delta        (0.5-2.75 Hz)
00331  *   1: Theta        (3.5-6.75 Hz)
00332  *   2: Low-Alpha    (7.5-9.25 Hz)
00333  *   3: High-Alpha   (10-11.75 Hz)
00334  *   4: Low-Beta     (13-16.75 Hz)
00335  *   5: High-Beta    (18-29.75 Hz)
00336  *   6: Low-Gamma    (31-39.75 Hz)
00337  *   7: High-Gamma   (41-49.75 Hz)
00338  *
00339  * @param meter array of meter data for different frequency bands
00340  * @param meterMin array of minimum recorded samples of each band
00341  * @param meterMax arrat if naximum recorded samples of each band
00342  */
00343 void meterData(int meter[8], int meterMin[8], int meterMax[8])
00344 {
00345     //first good signal?
00346     if (!connected) {
00347         connected=true;
00348         screen.fillrect(0,0,319,30,Black); //clear the Waiting to connect msg
00349         initialDelay.reset();
00350         initialDelay.start();
00351     }
00352     //minimal meter bars up at the top of the screen
00353     for (int j=0; j<8; j++) {
00354         displayBarGraph(5 + j * 13, 30, 16 + j * 13, 3, false, meter[j], meterMin[j], meterMax[j], RGBColour(0, j*2, 0x10), RGBColour(0, j*32, 0xFF), 0x00);
00355     }
00356     //Hijack this routine for menu movement
00357     if (started) {
00358         if (col==-1) {
00359             row++;
00360             if (row>=ROWS)
00361                 row=0;
00362         } else if (col==-2) {
00363             col=0;
00364         } else {
00365             col++;
00366             if (col>=keysRowSize[row])
00367                 col=-1;
00368         }
00369         drawKeyboard(keysRowSize, keys, keysColPos, row, col);
00370     }
00371 }
00372 
00373 /** This will be called when wave data arrives.
00374  * There will be a lot of these, 512 a second, so if you're planning to do anything
00375  * here, don't let it take long.  Best not to printf this out as it will just choke.
00376  *
00377  * param wave Raw wave data point
00378  */
00379 void waveData(int wave)
00380 {
00381 }
00382 
00383 //*****************
00384 //End User routines
00385 //*****************
00386 
00387 //System routines to obtain and parse data
00388 
00389 /** Simplify serial comms
00390  */
00391 unsigned char ReadOneByte()
00392 {
00393     int ByteRead;
00394 
00395     while(!blueSmirf.readable());
00396     ByteRead = blueSmirf.getc();
00397 
00398     return ByteRead;
00399 }
00400 
00401 /** Main loop, sets up and keeps listening for serial
00402  */
00403 int main()
00404 {
00405     //Video setup
00406     screen.claim(stdout);        // send stdout to the TFT display
00407     screen.background(Black);    // set background to black
00408     screen.foreground(White);    // set chars to white
00409     screen.cls();                // clear the screen
00410     screen.set_orientation(1);
00411     screen.set_font((unsigned char*) Arial12x12);
00412     screen.locate(5,5);
00413     screen.printf("Waiting to connect...");
00414 
00415     drawText();
00416     drawKeyboard(keysRowSize, keys, keysColPos, -1, -1);
00417 
00418     //Voice setup
00419     voice.baud(9600);
00420     voice.printf("\n");
00421     while (!voice.readable())
00422         ;
00423     wait(0.01);
00424     voice.getc();
00425     say("Welcome to Blink talk.");
00426 
00427     Timer t; //packet timer
00428     t.start();
00429     Timer blinkTimer; //used for detecting blinks
00430     int time;
00431     int generatedChecksum = 0;
00432     int checksum = 0;
00433     int payloadLength = 0;
00434     int payloadData[64] = {0};
00435     int poorQuality = 0;
00436     int attention = 0;
00437     int meditation = 0;
00438     int wave = 0;
00439     int meter[8] = {0};
00440     int meterMin[8];
00441     int meterMax[8];
00442     for (int j = 0; j < 8; j++) {
00443         meterMin[j]=99999999;
00444         meterMax[j]=-99999999;
00445     }
00446     bool eSensePacket = false;
00447     bool meterPacket = false;
00448     bool wavePacket = false;
00449 
00450     blueSmirf.baud(57600);
00451     blinkTimer.reset();
00452 
00453     while(1) {
00454         // Look for sync bytes
00455         if(ReadOneByte() == 170) {
00456             if(ReadOneByte() == 170) {
00457                 //Synchronised to start of packet
00458                 payloadLength = ReadOneByte();
00459                 if(payloadLength > 169) //Payload length can not be greater than 169
00460                     return;
00461 
00462                 generatedChecksum = 0;
00463                 for(int i = 0; i < payloadLength; i++) {
00464                     payloadData[i] = ReadOneByte();            //Read payload into memory
00465                     generatedChecksum += payloadData[i];
00466                 }
00467 
00468                 checksum = ReadOneByte();                      //Read checksum byte from stream
00469                 generatedChecksum = 255 - (generatedChecksum & 0xFF);   //Take one's compliment of generated checksum
00470 
00471                 if(checksum == generatedChecksum) {
00472                     //Packet seems OK
00473                     poorQuality = 200;
00474                     attention = 0;
00475                     meditation = 0;
00476                     wave = 0;
00477                     for(int i = 0; i < payloadLength; i++) {    // Parse the payload
00478                         switch (payloadData[i]) {
00479                             case 2: //quality
00480                                 i++;
00481                                 poorQuality = payloadData[i];
00482                                 eSensePacket = true;
00483                                 break;
00484                             case 4: //attention
00485                                 i++;
00486                                 attention = payloadData[i];
00487                                 eSensePacket = true;
00488                                 break;
00489                             case 5: //meditation
00490                                 i++;
00491                                 meditation = payloadData[i];
00492                                 eSensePacket = true;
00493                                 break;
00494                             case 0x80: //wave
00495                                 wave = payloadData[i+2] * 256 + payloadData[i+3];
00496                                 //We also want to try to detect blinks via analysing wave data
00497                                 time = blinkTimer.read_ms();
00498                                 if (wave > 32767) wave -= 65535; //cope with negatives
00499                                 if (wave>200 && time == 0) {
00500                                     blinkTimer.start();
00501                                 } else if (wave<-90 && time > 10 && time < 350) {
00502                                     blinkTimer.stop();
00503                                     blinkTimer.reset();
00504                                     blinked();
00505                                 } else if (time>500) {
00506                                     blinkTimer.stop();
00507                                     blinkTimer.reset();
00508                                 }
00509                                 i = i + 3;
00510                                 wavePacket = true;
00511                                 break;
00512                             case 0x83: //meter readings for different frequency bands
00513                                 for (int j=0; j<8; j++) {
00514                                     //documentation is inconsistent about whether these values are big-endian or little-endian,
00515                                     //and claims both in different places.  But wave data is big-endian so assuming that here.
00516                                     meter[j] = payloadData[i+j*3+2]*65536 + payloadData[i+j*3+3]*256 + payloadData[i+j*3+4];
00517                                     if (quality==0) {
00518                                         if (meter[j]<meterMin[j])
00519                                             meterMin[j]=meter[j];
00520                                         if (meter[j]>meterMax[j])
00521                                             meterMax[j]=meter[j];
00522                                     }
00523                                 }
00524                                 meterPacket = true;
00525                                 i = i + 25;
00526                                 break;
00527                             default:
00528                                 break;
00529                         } // switch
00530                     } // for loop
00531 
00532                     //Call routines to process data
00533                     if(eSensePacket) {
00534                         eSenseData(poorQuality, attention, meditation, t.read_ms());
00535                         eSensePacket = false;
00536                     }
00537                     if (meterPacket) {
00538                         meterData(meter, meterMin, meterMax);
00539                         t.reset();
00540                         meterPacket=false;
00541                     }
00542                     if (wavePacket) {
00543                         waveData(wave);
00544                         wavePacket=false;
00545                     }
00546                 } else {
00547                     // Checksum Error
00548                 }  // end if else for checksum
00549             } // end if read 0xAA byte
00550         } // end if read 0xAA byte
00551     } //end while
00552 }