androidのmic入力をnodeサーバ経由でplotlyに流して波形をモニタする。

androidのmicサンプリングデータをsocketioでnodeサーバに送る。
nodeサーバからplotlyにstreamingデータを送って波形をモニタする。

plotly
https://plot.ly/

android version 4.2 (xperiaA)
node version 0.10.26
express version 4.0.0
Android Studio(Beta) 0.8.9



PCで1kHzのサイン波を鳴らしながらキャプチャすると、こんな感じでみえます。

plotly

















---------
 plotly
---------
plotlyのサイトでユーザ登録。(無料)
・API key
・Streaming API token
をもらう。



------------
 node.js
------------
viewなしで、
android ; socketio
plotly:      http
で接続して、micサンプリングデータを中継する。

express4でひな形をつくり、以下を編集。
////////////////////////////////////////////////////////////////////////////////
--- package.json ---
{
  "name": "application-name",
  "version": "0.0.1",
  "private": true,
  "scripts": {
    "start": "node ./bin/www"
  },
  "dependencies": {
    "express": "~4.0.0",
    "static-favicon": "~1.0.0",
    "morgan": "~1.0.0",
    "cookie-parser": "~1.0.1",
    "body-parser": "~1.0.0",
    "debug": "~0.7.4",
    "jade": "~1.3.0",
        "plotly": "~0.2.13",
    "socket.io": "0.9.16"
  }
}
---------------------------------------------
"plotly": "~0.2.13",
"socket.io": "0.9.16"
を追加し、npm installする。

////////////////////////////////////////////////////////////////////////////////////////////////////////
--- bin/www ---
#!/usr/bin/env node
var debug = require('debug')('my-application');
var app = require('../app');


////////////////////////////////////////////////////////////////////////////////////////////////////////
--- app.js ----
赤塗りの部分はplotly関係の情報をいれる。
8kHzサンプルでデータを取り込んでいるので、x-parametaは1/8000step

var express = require('express');
var path = require('path');
var favicon = require('static-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');

var routes = require('./routes/index');
var users = require('./routes/users');

var plotly = require('plotly')('plotly_username','plotly_APIKEY');
var initData = [{x:[], y:[], stream:{token:'plotly_streamToken', maxpoints:100}}];
var initGraphOptions = {fileopt : "extend", filename : "plotly_filename"};
var myplot = require('./public/javascripts/myplotly');

var app = express();

// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');

app.use(favicon());
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded());
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));

app.use('/', routes);
app.use('/users', users);

app.use(function(req, res, next) {
    var err = new Error('Not Found');
    err.status = 404;
    next(err);
});

if (app.get('env') === 'development') {
    app.use(function(err, req, res, next) {
        res.status(err.status || 500);
        res.render('error', {
            message: err.message,
            error: err
        });
    });
}

app.use(function(err, req, res, next) {
    res.status(err.status || 500);
    res.render('error', {
        message: err.message,
        error: {}
    });
});

var http = require('http');
var server = http.createServer(app);
server.listen(3000);
var io = require('socket.io').listen(server);

//io.disable('heartbeats');
io.heartbeatTimeout = 100 * 1000;

var times=0;
var dataOn=0;
var jdata=[10];
var sendPage = 0;

plotly.plot(initData, initGraphOptions, function (err, msg) {
  if (err) return console.log(err);
  console.log(msg);
 
  var stream1 = plotly.stream('plotly_streamToken', function (err, res) {
    console.log(err, res);
        console.log('stream_finish');
    clearInterval(loop); // once stream is closed, stop writing
  });
  var i = 0;
    var ii = 0;
  var loop = setInterval(function () {
            if(dataOn !=0){
            if(i<jdata[sendPage].length){
                var j = jdata[sendPage][i];
          var streamObject = JSON.stringify({ x : ii*1/8000, y : j });
          stream1.write(streamObject+'\n');
          i++;
                ii++;
                if(i >= jdata[sendPage].length){
                    i = 0;
                    sendPage++;
                    if(sendPage >= 1){
                        dataOn = 0;
                        times = 0;
                        sendPage = 0;
                        i = 0;
                        ii = 0;
                    }
                }
               
            }
            }
  }, 20);
    stream1.setTimeout(5*60*1000);
    stream1.on('timeout', function(){
        console.log('user_timeout');   
        stream1.abort();
    });
    stream1.on('error', function(err){
        console.log('stream1_err:'+err.code);
    });
});


io.sockets.on('connection', function(socket) {
    console.log("connection");
    socket.on('message', function(data) {
                if(times < 1){
            console.log("message:"+data);
               jdata[times] = data;
                    times++;
                    dataOn = 1;
                }
    });
    socket.on('disconnect', function(){
        console.log("disconnect");
    });
});

module.exports = app;


//////////////////////////////////////////////////////////////////////////////////////////////////////////
---- public/javascript/myplotly.js ----

function myplot(){
plotly.plot(initData, initGraphOptions, function (err, msg) {
  if (err) return console.log(err)
  console.log(msg);
 
  var stream1 = plotly.stream('y4usa3f2hf', function (err, res) {
    console.log(err, res);
    clearInterval(loop); // once stream is closed, stop writing
  });
  var i = 0;
  var loop = setInterval(function () {
      var streamObject = JSON.stringify({ x : i, y : i });
      stream1.write(streamObject+'\n');
      i++;
            if(i==10){
                clearInterval(this);
            }
  }, 100);
});
}

var plot_clear = function(){
plotly.plot(initData, initGraphOptions, function (err, msg) {
  if (err) return console.log(err)
  console.log(msg);
 
  var stream1 = plotly.stream('y4usa3f2hf', function (err, res) {
    console.log(err, res);
    clearInterval(loop); // once stream is closed, stop writing
  });
  var i = 0;
  var loop = setInterval(function () {
            console.log('plotly_send');
      var streamObject = JSON.stringify({ x : i, y : 0 });
      stream1.write(streamObject+'\n');
      i++;
            if(i==10){
                stream1.destroy();
            }
  }, 100);
});
}

////////////////////////////////////////////////////////////////////////////


-------------
 android
------------
socketioのライブラリを使わせてもらって、nodeサーバと通信する。
https://github.com/hakamata/SocketIOSample




Screenshot_2014-12-02-16-21-16





























送信ボタンとレベルメータのみ。
ボタンを押した瞬間(450点くらい)の波形をnodeサーバに送信。


/////////////////////////////////////////////////////////////////////
---  src/main/AndroidManifest.xml ---
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
    package="info.kdapp.mymicview" >

    <uses-sdk android:minSdkVersion="7" />

    <uses-permission android:name="android.permission.RECORD_AUDIO" />
    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
    <uses-permission android:name="android.permission.INTERNET"/>

    <application
        android:allowBackup="true"
        android:icon="@drawable/ic_launcher"
        android:label="@string/app_name"
        android:theme="@style/AppTheme" >

        <activity
            android:name=".MyActivity"
            android:label="@string/app_name" >
            <intent-filter>
                <action android:name="android.intent.action.MAIN" />

                <category android:name="android.intent.category.LAUNCHER" />
            </intent-filter>
        </activity>
    </application>

</manifest>

//////////////////////////////////////////////////////////////////////////////////////////////////
--- src/main/java/..../MyActibity -----

サンプリングレートは8000

package info.kdapp.mymicview;
import android.app.Activity;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.os.Bundle;
import android.os.Environment;
import android.os.Handler;
import android.text.format.Time;
import android.util.Log;
import android.view.View;
import android.widget.Button;
import android.widget.ProgressBar;
import android.widget.Toast;

import org.json.JSONArray;
import org.json.JSONObject;

import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;

import io.socket.IOAcknowledge;
import io.socket.IOCallback;
import io.socket.SocketIO;
import io.socket.SocketIOException;

public class MyActivity extends Activity {

    public static final int SAMPLE_RATE = 8000;

    private AudioRecord mRecorder;
    private File mRecording;
    private short[] mBuffer;
    private final String btnLable = "send";
    private boolean mIsRecording = false;
    private ProgressBar mProgressBar;

    private SocketIO socket;
    private Handler handler = new Handler();
    private boolean sendOn;

    @Override
    public void onCreate(final Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_my);
        sendOn = false;
        initRecorder();

        mProgressBar = (ProgressBar) findViewById(R.id.progressBar);

        final Button button = (Button) findViewById(R.id.button);
        button.setText(btnLable);
        recOn();

        button.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(final View v) {
                if (!sendOn) {
                    sendOn = true;
                    test = 0;
                } else {
                    sendOn = false;
                }
            }
        });

        try {
            connect();
            Log.d("MIC", "connection pass");
        } catch (Exception e) {
            Log.d("MIC", "connection error");
            e.printStackTrace();
        }
    }
    private void recOn() {
        mIsRecording = true;
        mRecorder.startRecording();
        mRecording = getFile("raw");
        startBufferedWrite(mRecording);

    }
    private void recOff() {
        mIsRecording = false;
        mRecorder.stop();
    }
    private void recOntoOff() {
        sendOn = false;
    }

    @Override
    public void onDestroy() {
        mRecorder.release();
        super.onDestroy();
    }

    private void initRecorder() {
        int bufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO,
                AudioFormat.ENCODING_PCM_16BIT);
        mBuffer = new short[bufferSize];
        mRecorder = new AudioRecord(MediaRecorder.AudioSource.MIC, SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO,
                AudioFormat.ENCODING_PCM_16BIT, bufferSize);
    }

    private void startBufferedWrite(final File file) {
        new Thread(new Runnable() {
            @Override
            public void run() {
                DataOutputStream output = null;
                try {
                    Log.d("MIC","StartRec");
                    while (mIsRecording) {
                        double sum = 0;
                        int readSize = mRecorder.read(mBuffer, 0, mBuffer.length);
                        for (int i = 0; i < readSize; i++) {
                            sum += mBuffer[i] * mBuffer[i];
                        }
                        if (readSize > 0) {
                            final double amplitude = sum / readSize;
                            mProgressBar.setProgress((int) Math.sqrt(amplitude));
                        }
                        if(sendOn) {
                            sendEvent(readSize);
                        }
                    }
                } catch (Exception e) {
                    Log.d("MIC","errorRec");
                    Toast.makeText(MyActivity.this, e.getMessage(), Toast.LENGTH_SHORT).show();
                } finally {
                    mProgressBar.setProgress(0);
                }
            }
        }).start();
    }
    private File getFile(final String suffix) {
        Time time = new Time();
        time.setToNow();
        return new File(Environment.getExternalStorageDirectory(), time.format("%Y%m%d%H%M%S") + "." + suffix);
    }

    private void writeInt(final DataOutputStream output, final int value) throws IOException {
        output.write(value >> 0);
        output.write(value >> 8);
        output.write(value >> 16);
        output.write(value >> 24);
    }

    private void writeShort(final DataOutputStream output, final short value) throws IOException {
        output.write(value >> 0);
        output.write(value >> 8);
    }

    private void writeString(final DataOutputStream output, final String value) throws IOException {
        for (int i = 0; i < value.length(); i++) {
            output.write(value.charAt(i));
        }
    }

    private void connect() throws MalformedURLException {
        socket = new SocketIO("http://node_server_address:3000/");
        socket.connect(iocallback);
    }

    private IOCallback iocallback = new IOCallback() {

        @Override
        public void onConnect() {
            Log.d("MIC","onConnect");
        }

        @Override
        public void onDisconnect() {
            Log.d("Mic","onDisconnect");
        }

        @Override
        public void onMessage(JSONObject json, IOAcknowledge ack) {
            Log.d("MIC","onMesSer"+json+ack);
        }

        @Override
        public void onMessage(String data, IOAcknowledge ack) {
            Log.d("MIC","onMesSer"+data+ack);
        }

        @Override
        public void on(String event, IOAcknowledge ack, Object... args) {
            final JSONObject message = (JSONObject) args[0];

            new Thread(new Runnable() {
                public void run() {
                    handler.post(new Runnable() {
                        public void run() {
                            try {

                            } catch (Exception e) {
                                e.printStackTrace();
                            }
                        }
                    });
                }
            }).start();
        }

        @Override
        public void onError(SocketIOException socketIOException) {
            System.out.println("onError");
            Log.d("MIC","onError"+socketIOException);
            socketIOException.printStackTrace();
        }
    };

    int test = 0;

    public void sendEvent(int dnum) {
        try {
            if (test < 1) {
                JSONArray jjdata = new JSONArray();
                for (int i = 0; i < dnum; i++) {
                    jjdata.put(mBuffer[i]);
                }

                socket.emit("message", jjdata);
                test++;
                Log.d("MIC","emiton");
            }
        } catch (Exception ex) {
            Log.d("MIC","emitFail");
        } finally {
            recOntoOff();
        }
    }
}

//////////////////////////////////////////////////////////////////////

--- main/res/drawable/progressbar.xml ---

<layer-list xmlns:android="http://schemas.android.com/apk/res/android" >

    <item android:id="@android:id/background">
        <shape>
            <corners android:radius="8dp" />

            <gradient
                android:endColor="#000000"
                android:startColor="#000000" />

            <stroke
                android:width="1dp"
                android:color="#FFFFFF" />
        </shape>
    </item>
    <item android:id="@android:id/progress">
        <clip>
            <shape>
                <corners android:radius="8dp" />

                <gradient
                    android:endColor="#FF0000"
                    android:startColor="#FF8C00" />
            </shape>
        </clip>
    </item>

</layer-list>

///////////////////////////////////////////////////////////////

--- main/rees/layout/activity_my.xml ------

<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
    android:layout_width="fill_parent"
    android:layout_height="fill_parent"
    android:orientation="vertical" >

    <Button
        android:id="@+id/button"
        android:layout_width="fill_parent"
        android:layout_height="wrap_content"
        android:layout_margin="10dp" />

    <LinearLayout
        android:orientation="vertical"
        android:layout_width="fill_parent"
        android:layout_height="100px"></LinearLayout>


    <TextView
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:textAppearance="?android:attr/textAppearanceSmall"
        android:text="mic input level"
        android:id="@+id/textView" />

    <ProgressBar
        android:id="@+id/progressBar"
        style="?android:attr/progressBarStyleHorizontal"
        android:layout_width="fill_parent"
        android:layout_height="wrap_content"
        android:layout_margin="10dp"
        android:max="4000"
        android:progressDrawable="@drawable/progressbar" />

</LinearLayout>


///////////////////////////////////////////////////////////////////////////

main/res/values/strings.xml

<?xml version="1.0" encoding="utf-8"?>
<resources>

    <string name="app_name">micView</string>
    <string name="action_settings">menu</string>

</resources>


///////////////////////////////////////////////////////////////////////////////////

音源

こんなのでPCから1kHzのサイン波を鳴らしておいてサンプリング。
http://kdesignhp.web.fc2.com/colorcode/audio.html