Introduction
This week we explored how to get our application to have access to the microphone and speaker. We first drew samples from the microphone, then created a feedback-loop where the microphone samples were sent to the speaker, and finally learned how to record a chunk of audio using our previous button class as a “record” and “play” button. The code is copied below and requires the “iOS” version of openFrameworks. So working with the “emptyExample” of the iOS version (/of_preRelease_v007_iphone/apps/iPhoneExamples/emptyExample), try playing with the code below. Please have a read through the code and get it compiled on your own machine in your own time. Feel free to shoot me an e-mail if you have any questions. As a lot was covered, your homework is to explore the code in more depth and to understand it as much as possible.
Microphone Input
testApp.h
/*
* Created by Parag K. Mital - http://pkmital.com
* Contact: parag@pkmital.com
*
* Copyright 2011 Parag K. Mital. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#pragma once
#include "ofMain.h"
#include "ofxiPhone.h"
#include "ofxiPhoneExtras.h"
class testApp : public ofxiPhoneApp{
public:
void setup();
void update();
void draw();
// touch callbacks
void touchDown(ofTouchEventArgs &touch);
void touchMoved(ofTouchEventArgs &touch);
void touchUp(ofTouchEventArgs &touch);
void touchDoubleTap(ofTouchEventArgs &touch);
void touchCancelled(ofTouchEventArgs &touch);
// new method for getting the samples from the microphone
void audioIn( float * input, int bufferSize, int nChannels );
int width, height;
// variables which will help us deal with audio
int initialBufferSize;
int sampleRate;
float * buffer;
};
testApp.mm
/*
* Created by Parag K. Mital - http://pkmital.com
* Contact: parag@pkmital.com
*
* Copyright 2011 Parag K. Mital. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "testApp.h"
//--------------------------------------------------------------
void testApp::setup(){
// register touch events
ofRegisterTouchEvents(this);
// change our phone orientation to landscape
ofxiPhoneSetOrientation(OFXIPHONE_ORIENTATION_LANDSCAPE_RIGHT);
// variable for our audio
initialBufferSize = 512; // how many samples per "frame". each time "audioIn" gets called, we have this many samples
sampleRate = 44100; // how many samples per second. so sampleRate/initialBufferSize is our audio frame rate, or frames per second.
// we set our buffer pointer to a chunk of memory "initialBufferSize" large. this is called memory allocation.
buffer = new float[initialBufferSize];
// now we set that portion of memory to equal 0, or else it may be filled with garbage
memset(buffer, 0, initialBufferSize * sizeof(float));
// 0 output channels,
// 1 input channels
// 44100 samples per second
// 512 samples per buffer
// 4 num buffers (latency)
ofSoundStreamSetup(0, 1, this, sampleRate, initialBufferSize, 4);
ofSetFrameRate(60);
// resize our window
width = 480;
height = 320;
ofSetWindowShape(width, height);
}
//--------------------------------------------------------------
void testApp::update(){
}
//--------------------------------------------------------------
void testApp::draw(){
/*
// intialize a variable with 1 element only
float buffer;
buffer = 0;
// initialize an array of 512 elements
float buffer[512];
// can't write: buffer = 0;
buffer[0] = 0;
buffer[1] = 0;
...
...
buffer[511] = 0;
// create a pointer which points to memory.
// by default, this pointer is NULL, as in it doesn't point to any memory.
// then the right side of the equation allocates 512 elements in memory,
// and sets the pointer on the left side to "point" to that location in memory.
float *buffer = new float[512];
// we can "dereference" the pointer using the same notation as the array above
buffer[0] = 0; // equivalent to: *(buffer + 0) = 0;
buffer[1] = 0;
...
...
buffer[511] = 0;
*/
ofBackground(0);
// let's move halfway down the screen
ofTranslate(0, height/2);
// ratio of the screen size to the size of my buffer
float width_ratio = width / (float)initialBufferSize;
// change future drawing commands to be white
ofSetColor(255, 255, 255);
// we rescale our drawing to be 100x as big, since our audio waveform is only between [-1,1].
float amplitude = 100;
// lets draw the entire buffer by drawing tiny line segments from [0-1, 1-2, 2-3, 3-4, ... 510-511]
for (int i = 1; i < initialBufferSize; i++) { // we start at 1 instead of 0
ofLine((i-1)*width_ratio, // the first time this will be the x-value of the 0th buffer value
buffer[i-1]*amplitude, // the y-value of the 0th buffer value
i*width_ratio, // the x-value of the 1st buffer value
buffer[i]*amplitude); // the y-value of the 1st buffer value
}
}
//--------------------------------------------------------------
void testApp::audioIn(float * input, int bufferSize, int nChannels){
// just makes sure we didn't set "initialBufferSize" to something our audio card can't handle.
if( initialBufferSize != bufferSize ){
ofLog(OF_LOG_ERROR, "your buffer size was set to %i - but the stream needs a buffer size of %i", initialBufferSize, bufferSize);
return;
}
// we copy the samples from the "input" which is our microphone, to a variable
// our entire class knows about, buffer. this buffer has been allocated to have
// the same amount of memory as each "Frame" or "bufferSize" of audio has.
// so we copy the whole 512 sample chunk into our buffer so we can draw it.
for (int i = 0; i < bufferSize; i++){
buffer[i] = input[i];
}
}
//--------------------------------------------------------------
void testApp::touchDown(ofTouchEventArgs &touch) {}
void testApp::touchMoved(ofTouchEventArgs &touch) {}
void testApp::touchUp(ofTouchEventArgs &touch) {}
void testApp::touchDoubleTap(ofTouchEventArgs &touch) {}
void testApp::touchCancelled(ofTouchEventArgs &touch) {}
Recording Sound
For this example, we want to record a bit of sound using our button class to help us. We need to copy the button classes files (padButton.h/padButton.cpp) to our project, and also remember to add the images (button.png/button-down.png) to our data folder. We also modified our button class so that the “pressed” and “released” functions return a boolean value. This code is copied below.
testApp.h
/*
* Created by Parag K. Mital - http://pkmital.com
* Contact: parag@pkmital.com
*
* Copyright 2011 Parag K. Mital. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#pragma once
#include "ofMain.h"
#include "ofxiPhone.h"
#include "ofxiPhoneExtras.h"
#include "padButton.h"
class testApp : public ofxiPhoneApp{
public:
void setup();
void update();
void draw();
void touchDown(ofTouchEventArgs &touch);
void touchMoved(ofTouchEventArgs &touch);
void touchUp(ofTouchEventArgs &touch);
void touchDoubleTap(ofTouchEventArgs &touch);
void touchCancelled(ofTouchEventArgs &touch);
void audioIn( float * input, int bufferSize, int nChannels );
void audioOut( float * output, int bufferSize, int nChannels );
// our screen size
int width, height;
// variable for audio
int initialBufferSize;
int sampleRate;
// this vector will store our audio recording
vector<float> buffer;
// frame will tell us "what chunk of audio are we currently playing back".
// as we record and play back in "chunks" also called "frames", we will need
// to keep track of which frame we are playing during playback
int frame;
// frame will run until it hits the final recorded frame, which is numFrames
// we increment this value every time we record a new frame of audio
int numFrames;
// our buttons for user interaction
padButton button_play, button_record;
// determined based on whether the user has pressed the play or record buttons
bool bRecording, bPlaying;
};
testApp.mm
/*
* Created by Parag K. Mital - http://pkmital.com
* Contact: parag@pkmital.com
*
* Copyright 2011 Parag K. Mital. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "testApp.h"
//--------------------------------------------------------------
void testApp::setup(){
// register touch events
ofRegisterTouchEvents(this);
ofxiPhoneSetOrientation(OFXIPHONE_ORIENTATION_LANDSCAPE_RIGHT);
//for some reason on the iphone simulator 256 doesn't work - it comes in as 512!
//so we do 512 - otherwise we crash
initialBufferSize = 512;
sampleRate = 44100;
//buffer = new float[initialBufferSize];
//memset(buffer, 0, initialBufferSize * sizeof(float));
// 1 output channels, <-- different from the previous example, as now we want to playback
// 1 input channels
// 44100 samples per second
// 512 samples per buffer
// 4 num buffers (latency)
ofSoundStreamSetup(1, 1, this, sampleRate, initialBufferSize, 4);
ofSetFrameRate(60);
// we initialize our two buttons to act as a play and a record button.
button_record.loadImages("button.png", "button-down.png");
button_play.loadImages("button.png", "button-down.png");
button_record.setPosition(100, 100);
button_record.setSize(100, 100);
button_play.setPosition(250, 100);
button_play.setSize(100, 100);
// initially not recording
bRecording = false;
bPlaying = false;
// which audio frame am i currently playing back
frame = 0;
// how many audio frames did we record?
numFrames = 0;
width = 480;
height = 320;
ofSetWindowShape(width, height);
}
//--------------------------------------------------------------
void testApp::update(){
}
//--------------------------------------------------------------
void testApp::draw(){
ofBackground(0);
// draw our buttons
button_record.draw();
button_play.draw();
ofTranslate(0, height/2);
// let's output a string on the screen that helps us determine what the state of the program is
if (bPlaying) {
ofDrawBitmapString("Playing: True", 20,20);
}
else {
ofDrawBitmapString("Playing: False", 20,20);
}
if (bRecording) {
ofDrawBitmapString("Recording: True", 20,40);
}
else {
ofDrawBitmapString("Recording: False", 20,40);
}
}
//--------------------------------------------------------------
void testApp::audioOut(float * output, int bufferSize, int nChannels){
// if we are playing back audio (if the user is pressing the play button)
if(bPlaying)
{
// we set the output to be our recorded buffer
for (int i = 0; i < bufferSize; i++){
// so we have to access the current "playback frame" which is a variable
// "frame". this variable helps us determine which frame we should play back.
// because one frame is only 512 samples, or 1/90th of a second of audio, we would like
// to hear more than just that one frame. so we playback not just the first frame,
// but every frame after that... after 90 frames of audio, we will have heard
// 1 second of the recording...
output[i] = buffer[i + frame*bufferSize];
}
// we have to increase our frame counter in order to hear farther into the audio recording
frame = (frame + 1) % numFrames;
}
// else don't output anything to the speaker
else {
memset(output, 0, nChannels * bufferSize * sizeof(float));
}
}
//--------------------------------------------------------------
void testApp::audioIn(float * input, int bufferSize, int nChannels){
if( initialBufferSize != bufferSize ){
ofLog(OF_LOG_ERROR, "your buffer size was set to %i - but the stream needs a buffer size of %i", initialBufferSize, bufferSize);
return;
}
// if we are recording
if(bRecording)
{
// let's add the current frame of audio input to our recording buffer. this is 512 samples.
// (note: another way to do this is to copy the whole chunk of memory using memcpy)
for (int i = 0; i < bufferSize; i++)
{
// we will add a sample at a time to the back of the buffer, increasing the size of "buffer"
buffer.push_back(input[i]);
}
// we also need to keep track of how many audio "frames" we have. this is how many times
// we have recorded a chunk of 512 samples. we refer to that chunk of 512 samples as 1 frame.
numFrames++;
}
// otherwise we set the input to 0
else
{
// set the chunk in memory pointed to by "input" to 0. the
// size of the chunk is the 3rd argument.
memset(input, 0, nChannels * bufferSize * sizeof(float));
}
}
//--------------------------------------------------------------
void testApp::touchDown(ofTouchEventArgs &touch){
//bRecording = !bRecording;
// NOTE: we have modified our button class to return true or false when the button was pressed
// we set playing or recording to be true depending on whether the user pressed that button
bPlaying = button_play.pressed(touch.x, touch.y);
bRecording = button_record.pressed(touch.x, touch.y);
}
//--------------------------------------------------------------
void testApp::touchMoved(ofTouchEventArgs &touch){
}
//--------------------------------------------------------------
void testApp::touchUp(ofTouchEventArgs &touch){
// we then user stops pressing the button, we are no longer playing or recording
bPlaying = !button_play.released(touch.x, touch.y);
bRecording = !button_record.released(touch.x, touch.y);
}
//--------------------------------------------------------------
void testApp::touchDoubleTap(ofTouchEventArgs &touch){
}
//--------------------------------------------------------------
void testApp::touchCancelled(ofTouchEventArgs& args){
}
padButton.h
/*
* Created by Parag K. Mital - http://pkmital.com
* Contact: parag@pkmital.com
*
* Copyright 2011 Parag K. Mital. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
// include file only once
#pragma once
// include openframeworks
#include "ofMain.h"
// declare the padbutton class and methods
class padButton {
public:
// enumerators allow us to assign more interesting names to values of an integer
// we could use an integer to the same effect,
// e.g. "int button_state = 0", when our button is down
// and "int button_state = 1", when our button is normal,
// but enumerators allow us to instead say
// "BUTTON_STATE button_state = BUTTON_DOWN", when our button is down,
// "BUTTON_STATE button_state = NORMAL", when our button is normal.
enum BUTTON_STATE {
BUTTON_DOWN,
NORMAL
};
// default constructor - no parameters, no return type
padButton();
// methods which our button class will define
// one for loading images for each of the button states
void loadImages(string state_normal, string state_down);
// setters, to set internal variables
// the position
void setPosition(int x, int y);
// the size
void setSize(int w, int h);
// drawing the buttons
void draw();
// and interaction with the button
bool pressed(int x, int y);
bool released(int x, int y);
private:
// images for drawing
ofImage button_image_normal, button_image_down;
// our position
int button_x, button_y;
// size
int button_width, button_height;
// and internal state of the button
BUTTON_STATE button_state;
};
padButton.cpp
/*
* Created by Parag K. Mital - http://pkmital.com
* Contact: parag@pkmital.com
*
* Copyright 2011 Parag K. Mital. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include <iostream>
#include "padButton.h"
// default constructor definition
padButton::padButton()
{
// by default, we have a very boring button
button_x = 0;
button_y = 0;
button_width = 0;
button_height = 0;
button_state = NORMAL;
}
void padButton::loadImages(string state_normal, string state_down)
{
// load the images for our buttons
button_image_normal.loadImage(state_normal);
button_image_down.loadImage(state_down);
}
void padButton::setPosition(int x, int y)
{
// set our internal variables
button_x = x;
button_y = y;
}
void padButton::setSize(int w, int h)
{
// set our internal variables
button_width = w;
button_height = h;
}
void padButton::draw()
{
// allow alpha transparency
ofEnableAlphaBlending();
// if our button is normal
if(button_state == NORMAL)
{
// draw the normal button image
button_image_normal.draw(button_x,
button_y,
button_width,
button_height);
}
else
{
// draw the down image
button_image_down.draw(button_x,
button_y,
button_width,
button_height);
}
// ok done w/ alpha blending
ofDisableAlphaBlending();
}
bool padButton::pressed(int x, int y)
{
// compound boolean expressions to determine,
// is our x,y input within the bounds of the button
// we have to check the left, the top, the right, and the bottom sides
// of the button, respectively.
if( x > button_x && y > button_y
&& x < (button_x + button_width)
&& y < (button_y + button_height) )
{
button_state = BUTTON_DOWN;
// return yes since the user pressed the button
return true;
}
else {
// no the user didn't press the button
return false;
}
}
bool padButton::released(int x, int y)
{
// ok back to normal
button_state = NORMAL;
// we always return true since that is how our buttons work.
return true;
}