|
|
|
@ -1,33 +1,3 @@
|
|
|
|
|
/* Development Version
|
|
|
|
|
Created By: Seung Ah Lee
|
|
|
|
|
Modified by: Karina Samuel-Gama
|
|
|
|
|
Modified by: Amy Lam
|
|
|
|
|
|
|
|
|
|
This program runs patterned light stimuli. Users choose between different stimuli which
|
|
|
|
|
are projected through the projector and displayed onto Euglena. A webcam records the
|
|
|
|
|
projections overlayed with the view of the Euglena.
|
|
|
|
|
Current Functionality
|
|
|
|
|
- Screen calibration. Users can line up the 4 colored dots with the 4 corners of the screen
|
|
|
|
|
using the directional keys and home, end, page up, and page down
|
|
|
|
|
- Allows free draw. Users can change colors and pen width to interact with the euglena.
|
|
|
|
|
Users can also activate the four LEDs using 'a' 'w' 'd' 's'. In free draw users can also
|
|
|
|
|
see the computer tracking the top right corner of the screen by hitting CTRL
|
|
|
|
|
- Users can input LED sequences and see the reaction of the euglena. This allows for the
|
|
|
|
|
user to easily draw while the sequence is running. [The code can also be saved for future
|
|
|
|
|
functionality]
|
|
|
|
|
- The User can also save LED sequences and paint drawings for future uses. The data is only saved
|
|
|
|
|
in the current instance of the program. NEED to integrate the program with remote database
|
|
|
|
|
|
|
|
|
|
___________________________________________________________
|
|
|
|
|
| | |
|
|
|
|
|
| | Projection |
|
|
|
|
|
| Display Screen | Off Screen doodle |
|
|
|
|
|
| visible on laptop | appears here |
|
|
|
|
|
| | |
|
|
|
|
|
| | |
|
|
|
|
|
|___________________________|_____________________________|
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
import org.firmata.*;
|
|
|
|
|
import gab.opencv.*;
|
|
|
|
|
import processing.video.*;
|
|
|
|
@ -35,27 +5,14 @@ import processing.serial.*;
|
|
|
|
|
import cc.arduino.*;
|
|
|
|
|
import controlP5.*;
|
|
|
|
|
|
|
|
|
|
Capture cam; // One camera object needed to display euglena
|
|
|
|
|
Arduino arduino; // One arduino to control the LEDs
|
|
|
|
|
ControlP5 cp5; // Library to create nicer GUI
|
|
|
|
|
OpenCV opencv; // One opencv object needed per region of interest with unique dimension
|
|
|
|
|
OpenCV opencv2;
|
|
|
|
|
OpenCV opencv3;
|
|
|
|
|
OpenCV opencv4;
|
|
|
|
|
OpenCV opencv5;
|
|
|
|
|
|
|
|
|
|
/* CONSTANTS */
|
|
|
|
|
/////////////////// Program Controls
|
|
|
|
|
final long ledDelay = 5000; // Duration of light from LED during LED sequence
|
|
|
|
|
final int recordDelay = 0; // Time delay befoe taking next xcoord in case of lag
|
|
|
|
|
final int paintSize = 100; // Number of xcoord taken in limited paint activity
|
|
|
|
|
final int ledLimit = 3; // Number of LEDs in sequence
|
|
|
|
|
|
|
|
|
|
////////////////// LED Controls
|
|
|
|
|
final int left = 10;
|
|
|
|
|
final int right = 5;
|
|
|
|
|
final int down = 9;
|
|
|
|
|
final int up = 6;
|
|
|
|
|
|
|
|
|
|
//////////////////// Image Processing for doodle exploration
|
|
|
|
|
final int roiWidth = 500; // Width of ROI
|
|
|
|
@ -67,22 +24,8 @@ final int roiCornerY = 290;
|
|
|
|
|
final int centerY = 540; // Center Y coordinate
|
|
|
|
|
final int centerX = 860; // Center X coordinate
|
|
|
|
|
|
|
|
|
|
/////////////////// Game Controls
|
|
|
|
|
final int goalEntrance = 150; // 250 Width of entrance of goal
|
|
|
|
|
final int goalDepth = 150; // 100 Depth of goal
|
|
|
|
|
final float totalGameTime = 45000; // Duration of game
|
|
|
|
|
final float scoreDelay = 100; // time to wait before score update so score display doesn't look as jumpy
|
|
|
|
|
final float clearTime = 100000; // Duration of clear function : 1 minute 40 seconds
|
|
|
|
|
final String gameInstructions = "Pulsed 488 nm light for whole screen"; // Instructions
|
|
|
|
|
final String game2Instructions = "Draw barriers and use a, w, d, s to change the driection to get the euglena into the green box.";
|
|
|
|
|
final String game3Instructions = "Draw barriers to get the eugelna into the green box. Don't let them escape. You have limited paint.";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/* Global Variables */
|
|
|
|
|
///////////////// Miscellaneaous Variables
|
|
|
|
|
int penWidth = 15; // Default width of pen
|
|
|
|
|
int ledPin = 5; // LED of interest
|
|
|
|
|
boolean load = false; // Boolean to determine if user wants to load previously saved led sequences or paint
|
|
|
|
|
long ledTime; // Temporary time storage used to keep LED's on for certain period of time
|
|
|
|
|
long paintTime; // Temporary time storage used to determine when paint "runs out"
|
|
|
|
@ -97,15 +40,9 @@ int count; // Equivalent to number of outline
|
|
|
|
|
///////////////////////// Color Variables
|
|
|
|
|
int iColor=0;//current color
|
|
|
|
|
|
|
|
|
|
int rightArrow = 0;
|
|
|
|
|
int downArrow = 0;
|
|
|
|
|
int leftArrow = 0;
|
|
|
|
|
int upArrow = 0;
|
|
|
|
|
|
|
|
|
|
float lagTime = 1;
|
|
|
|
|
int totalTime = 15000;
|
|
|
|
|
|
|
|
|
|
int presetProg;
|
|
|
|
|
|
|
|
|
|
//////////////////// Storage Variables
|
|
|
|
|
IntList paintLimit = new IntList(); // Stores finite limit of XCoord of points drawn. Once limit is reached "drawing" is stopped
|
|
|
|
@ -131,41 +68,6 @@ PImage goal2;
|
|
|
|
|
PImage gameScreen;
|
|
|
|
|
PImage entireScreen;
|
|
|
|
|
PImage cvscreen;
|
|
|
|
|
// score handling
|
|
|
|
|
int plusScore; // Equivalent to number of outlines
|
|
|
|
|
int minusScore;
|
|
|
|
|
int totalScore;
|
|
|
|
|
float scoreTime; // Temporary variable to ensure score is displayed for proper length of time
|
|
|
|
|
float scoreUpdateTime; // Temporary variable to ensure score changes at even intervals
|
|
|
|
|
// Game time handling
|
|
|
|
|
float currentGameTime; // Temporary variable to ensure game runs for proper length of time
|
|
|
|
|
float secondCounter;
|
|
|
|
|
// screen clearing
|
|
|
|
|
float clearScreenTime; // Temporary variable to ensure screen clearing lasts for proper length of time
|
|
|
|
|
boolean cleared = false; // Keeps track of whether screen has been cleared
|
|
|
|
|
// LED control
|
|
|
|
|
int gamePin = 10; // Used to keep track of activated LED pin serves to limit user control over LED's
|
|
|
|
|
|
|
|
|
|
/////////////////////////Program Session Variables
|
|
|
|
|
int startSession = 0; // 0: "play" button, 1: menu, 2: ledSequence, 3: limitedPaint, 4: Level developer, 5: doodling, 6:Game Menu, 7: Level 1, 8: Level 2
|
|
|
|
|
int ledSession = 0; // 0: pick LED sequence, 1: run through sequence, 2: reset activity
|
|
|
|
|
int paintSession = 0; // 0: initialize Paint, 1: Run through paint session, 2: reset activity
|
|
|
|
|
int levelSession = 1; // 0: finish development, 1: Choose LED level or paint level, 2 : paint level, 3 : led level
|
|
|
|
|
int gameSession = 0; // 0: instructions and screen clear, 1: play level, 2: score game, 3: reset first level
|
|
|
|
|
int game2Session = 0; // 0: instructions and screenclear, 1: play level, 2: score game, 3: reset second level
|
|
|
|
|
int game3Session = 0; // 0: instructions and screenclear, 1: play level, 2: score game, 3: reset third level
|
|
|
|
|
int protoGameSession = 0;
|
|
|
|
|
|
|
|
|
|
int screenWidth;
|
|
|
|
|
int screenHeight;
|
|
|
|
|
int totalCount = 1;
|
|
|
|
|
float percentDensity;
|
|
|
|
|
int densityTSTime;
|
|
|
|
|
int densityROITime;
|
|
|
|
|
int displayDensityTime;
|
|
|
|
|
int densitySession = 0;
|
|
|
|
|
int gameScreenWidth = 600;
|
|
|
|
|
int gameScreenHeight = 600;
|
|
|
|
|
|
|
|
|
|
static float densityThreshold = 6;
|
|
|
|
|
boolean densityMeasured = false;
|
|
|
|
@ -173,83 +75,18 @@ boolean densityMeasured = false;
|
|
|
|
|
FloatList fadeXCoord = new FloatList(); // Stores x-coordinate of each point
|
|
|
|
|
FloatList fadeYCoord = new FloatList();
|
|
|
|
|
|
|
|
|
|
DropdownList presetsList;
|
|
|
|
|
|
|
|
|
|
ArrayList<Line> lines = new ArrayList<Line>();
|
|
|
|
|
ArrayList<Ellipse> ellipses = new ArrayList<Ellipse>();
|
|
|
|
|
ArrayList<Rectangle> rectangles = new ArrayList<Rectangle>();
|
|
|
|
|
ArrayList<Triangle> triangles = new ArrayList<Triangle>();
|
|
|
|
|
|
|
|
|
|
ArrayList<ShrinkWindow> shrinkwindows = new ArrayList<ShrinkWindow>();
|
|
|
|
|
ArrayList<ExpandWindow> expandwindows = new ArrayList<ExpandWindow>();
|
|
|
|
|
ArrayList<TranslateWindow> translatewindows = new ArrayList<TranslateWindow>();
|
|
|
|
|
ArrayList<RotateWindow> rotatewindows = new ArrayList<RotateWindow>();
|
|
|
|
|
|
|
|
|
|
EllipseDrawer ellipseDrawer = new EllipseDrawer();
|
|
|
|
|
LineDrawer lineDrawer = new LineDrawer();
|
|
|
|
|
RectangleDrawer rectangleDrawer = new RectangleDrawer();
|
|
|
|
|
TriangleDrawer triangleDrawer = new TriangleDrawer();
|
|
|
|
|
|
|
|
|
|
Shrink shrink = new Shrink();
|
|
|
|
|
Expand expand = new Expand();
|
|
|
|
|
Translate translate = new Translate();
|
|
|
|
|
Rotate rotate = new Rotate();
|
|
|
|
|
|
|
|
|
|
Calibrator calibrator = new Calibrator();
|
|
|
|
|
Menu menu;
|
|
|
|
|
|
|
|
|
|
char drawtype = 'd';
|
|
|
|
|
int preset;
|
|
|
|
|
|
|
|
|
|
void setup() {
|
|
|
|
|
count = 1;
|
|
|
|
|
totalCount = 1;
|
|
|
|
|
background(200);
|
|
|
|
|
size(displayWidth*2, displayHeight*2);
|
|
|
|
|
screenWidth = displayWidth;
|
|
|
|
|
screenHeight = displayHeight;
|
|
|
|
|
|
|
|
|
|
ellipseMode(CORNER);
|
|
|
|
|
smooth();
|
|
|
|
|
/*
|
|
|
|
|
// Capture cam - initialize correct camera
|
|
|
|
|
String[] cameras = Capture.list();
|
|
|
|
|
if (cameras.length == 0) {
|
|
|
|
|
println("There are no cameras available for capture.");
|
|
|
|
|
exit();
|
|
|
|
|
} else {
|
|
|
|
|
println("Available cameras:");
|
|
|
|
|
for(int i = 0; i < cameras.length; i++) {
|
|
|
|
|
println(i, cameras[i]);
|
|
|
|
|
}
|
|
|
|
|
println(cameras[37]);
|
|
|
|
|
cam = new Capture(this, cameras[69]); //cameras[39] : 1280x1024 fps=10, cameras[37] 1280 X 800 fps=25
|
|
|
|
|
cam.start();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Arduino - initialize correct arduino
|
|
|
|
|
String[] arduinos = Arduino.list();
|
|
|
|
|
if (arduinos.length == 0) {
|
|
|
|
|
println("There are no arduinos available for use.");
|
|
|
|
|
exit();
|
|
|
|
|
} else {
|
|
|
|
|
println("Available arduinos:");
|
|
|
|
|
for(int i = 0; i < arduinos.length; i++) {
|
|
|
|
|
println(arduinos[i]);
|
|
|
|
|
}
|
|
|
|
|
println(arduinos[0]);
|
|
|
|
|
arduino = new Arduino(this, arduinos[0], 57600); // arduinos[1] : COM4
|
|
|
|
|
arduino.pinMode(ledPin, Arduino.OUTPUT);
|
|
|
|
|
}
|
|
|
|
|
*/
|
|
|
|
|
// OpenCV - initialize correct frames and images
|
|
|
|
|
// opencv = new OpenCV(this, roiWidth, roiHeight);
|
|
|
|
|
// opencv.startBackgroundSubtraction(7, 3, .35);
|
|
|
|
|
// roiFrame = new PImage(roiWidth, roiHeight);
|
|
|
|
|
|
|
|
|
|
menu = new Menu(this);
|
|
|
|
|
menu.drawBackground();
|
|
|
|
|
menu.drawArrows();
|
|
|
|
|
|
|
|
|
|
clearDisplay();
|
|
|
|
|
}
|
|
|
|
@ -257,14 +94,6 @@ void setup() {
|
|
|
|
|
/* Runs contionous display of mouse in the background; keeps
|
|
|
|
|
track of what mode app is in and where mouse is on the screen */
|
|
|
|
|
void draw() {
|
|
|
|
|
// camera display - webcam feed reads into background of laptop display
|
|
|
|
|
if(cam.available() == true) {
|
|
|
|
|
cam.read();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
imageMode(CORNER);
|
|
|
|
|
image(cam, 0, 0, displayWidth - menu.width, displayHeight);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (showcv) { // Show image tracking when CTRL is pressed
|
|
|
|
|
identifyEuglena();
|
|
|
|
@ -274,10 +103,6 @@ void draw() {
|
|
|
|
|
recordTimelapse(lagTime, totalTime); //Note: the maximum fps is around 5
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
calibrator.draw();
|
|
|
|
|
menu.draw();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for (int i=0; i < shrinkwindows.size(); i++) {
|
|
|
|
|
shrinkwindows.get(i).draw();
|
|
|
|
|
}
|
|
|
|
@ -302,22 +127,6 @@ and off with standard directional keys. It also allows calibration of
|
|
|
|
|
the projectorApplet field of view */
|
|
|
|
|
|
|
|
|
|
void keyPressed() {
|
|
|
|
|
if (key == CODED) {
|
|
|
|
|
// Calibration of projectorApplet field of view
|
|
|
|
|
calibrator.buttonPressed(keyCode);
|
|
|
|
|
|
|
|
|
|
switch(keyCode) {
|
|
|
|
|
case 112: // F1 key
|
|
|
|
|
if (startSession == 8) { // Level 2
|
|
|
|
|
if(gamePin == 5) {
|
|
|
|
|
gamePin = 10; // Euglena move left
|
|
|
|
|
arduino.digitalWrite(right, Arduino.LOW);
|
|
|
|
|
} else {
|
|
|
|
|
gamePin = 5; // Euglena move right
|
|
|
|
|
arduino.digitalWrite(left, Arduino.LOW);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case CONTROL:
|
|
|
|
|
if(showcv) showcv = false;
|
|
|
|
@ -331,30 +140,6 @@ void keyPressed() {
|
|
|
|
|
|
|
|
|
|
// Control LED with standard directional keys
|
|
|
|
|
switch(key) {
|
|
|
|
|
case 'd':
|
|
|
|
|
ledPin = right; // Forces Euglena right
|
|
|
|
|
rightArrow = 204;
|
|
|
|
|
arduino.digitalWrite(ledPin, Arduino.HIGH);
|
|
|
|
|
break;
|
|
|
|
|
case 'w':
|
|
|
|
|
ledPin = up; // Forces Euglena up
|
|
|
|
|
upArrow = 204;
|
|
|
|
|
arduino.digitalWrite(ledPin, Arduino.HIGH);
|
|
|
|
|
break;
|
|
|
|
|
case 's':
|
|
|
|
|
ledPin = down; // Forces Euglena down
|
|
|
|
|
downArrow = 204;
|
|
|
|
|
arduino.digitalWrite(ledPin, Arduino.HIGH);
|
|
|
|
|
break;
|
|
|
|
|
case 'a':
|
|
|
|
|
ledPin = left; //Forces Euglena left
|
|
|
|
|
leftArrow = 204;
|
|
|
|
|
arduino.digitalWrite(ledPin, Arduino.HIGH);
|
|
|
|
|
break;
|
|
|
|
|
default: // Turn off last light
|
|
|
|
|
arduino.digitalWrite(ledPin, Arduino.LOW);
|
|
|
|
|
leftArrow = rightArrow = upArrow = downArrow = 0;
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case ENTER: // Takes a snapshot of the FOV
|
|
|
|
|
snapshot();
|
|
|
|
@ -366,41 +151,6 @@ void keyPressed() {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Turns off LED's that have been turned on while the app is running*/
|
|
|
|
|
void keyReleased() {
|
|
|
|
|
switch(key) {
|
|
|
|
|
case 'd':
|
|
|
|
|
ledPin = right;
|
|
|
|
|
rightArrow = 0;
|
|
|
|
|
arduino.digitalWrite(ledPin, Arduino.LOW);
|
|
|
|
|
break;
|
|
|
|
|
case 'w':
|
|
|
|
|
ledPin = up;
|
|
|
|
|
upArrow = 0;
|
|
|
|
|
arduino.digitalWrite(ledPin, Arduino.LOW);
|
|
|
|
|
break;
|
|
|
|
|
case 's':
|
|
|
|
|
ledPin = down;
|
|
|
|
|
downArrow = 0;
|
|
|
|
|
arduino.digitalWrite(ledPin, Arduino.LOW);
|
|
|
|
|
break;
|
|
|
|
|
case 'a':
|
|
|
|
|
ledPin = left;
|
|
|
|
|
leftArrow = 0;
|
|
|
|
|
arduino.digitalWrite(ledPin, Arduino.LOW);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
// case 'n':
|
|
|
|
|
// opencv = new OpenCV(this, roiWidth, roiHeight);
|
|
|
|
|
// opencv.startBackgroundSubtraction(7, 3, .35);
|
|
|
|
|
// roiFrame = new PImage(roiWidth, roiHeight);
|
|
|
|
|
// break;
|
|
|
|
|
default :
|
|
|
|
|
arduino.digitalWrite(ledPin, Arduino.LOW);
|
|
|
|
|
leftArrow = rightArrow = upArrow = downArrow = 0;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Listens for user dragging mouse and draws points of appropriate
|
|
|
|
|
color and width on the screen */
|
|
|
|
@ -413,68 +163,6 @@ void mouseDragged() {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void mouseClicked(){
|
|
|
|
|
if (mouseX < displayWidth-menu.width){
|
|
|
|
|
switch(drawtype) {
|
|
|
|
|
|
|
|
|
|
case 'e' :
|
|
|
|
|
ellipseDrawer.mouseClicked(mouseX, mouseY);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case 'l' :
|
|
|
|
|
lineDrawer.mouseClicked(mouseX, mouseY);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case 'r' :
|
|
|
|
|
rectangleDrawer.mouseClicked(mouseX, mouseY);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case 't' :
|
|
|
|
|
triangleDrawer.mouseClicked(mouseX, mouseY);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case 's' :
|
|
|
|
|
shrink.mouseClicked(mouseX, mouseY);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case 'x' :
|
|
|
|
|
expand.mouseClicked(mouseX, mouseY);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case 'm' :
|
|
|
|
|
translate.mouseClicked(mouseX, mouseY);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case 'c' :
|
|
|
|
|
rotate.mouseClicked(mouseX, mouseY);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
void controlEvent(ControlEvent theEvent) {
|
|
|
|
|
menu.controlEvent(theEvent);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Translates x coordinate of point from user screen to projection screeen */
|
|
|
|
|
float convertXCoord(float x) {
|
|
|
|
|
return (x / calibrator.magx + displayWidth * calibrator.offsetx);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Translates y coordinate of point from user screen to projection screen */
|
|
|
|
|
float convertYCoord(float y) {
|
|
|
|
|
return (y / calibrator.magy + displayHeight * calibrator.offsety);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Translates x distances on screen to projectorApplet*/
|
|
|
|
|
float convertXDistance(float x) {
|
|
|
|
|
return (x / calibrator.magx);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Translates y distances on screen to projectorApplet*/
|
|
|
|
|
float convertYDistance(float y) {
|
|
|
|
|
return (y / calibrator.magy);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|