/*
* Copyright 2017 The Android Things Samples Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.example.androidthings.imageclassifier;
import android.app.Activity;
import android.content.Context;
import android.content.pm.PackageManager;
import android.graphics.Bitmap;
import android.hardware.camera2.CameraAccessException;
import android.media.Image;
import android.media.ImageReader;
import android.os.Bundle;
import android.os.Handler;
import android.os.HandlerThread;
import android.speech.tts.TextToSpeech;
import android.speech.tts.UtteranceProgressListener;
import android.util.Log;
import android.util.Size;
import android.view.KeyEvent;
import android.view.View;
import android.view.WindowManager;
import android.widget.ImageView;
import android.widget.TextView;
import com.example.androidthings.imageclassifier.classifier.Recognition;
import com.example.androidthings.imageclassifier.classifier.TensorFlowImageClassifier;
import com.google.android.things.contrib.driver.button.Button;
import com.google.android.things.contrib.driver.button.ButtonInputDriver;
import com.google.android.things.pio.Gpio;
import com.google.android.things.pio.PeripheralManager;
import java.io.IOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.Locale;
import java.util.concurrent.atomic.AtomicBoolean;
public class ImageClassifierActivity extends Activity implements ImageReader.OnImageAvailableListener {
private static final String TAG = "ImageClassifierActivity";
// Matches the images used to train the TensorFlow model
private static final Size MODEL_IMAGE_SIZE = new Size(224, 224);
/* Key code used by GPIO button to trigger image capture */
private static final int SHUTTER_KEYCODE = KeyEvent.KEYCODE_CAMERA;
private ImagePreprocessor mImagePreprocessor;
private TextToSpeech mTtsEngine;
private TtsSpeaker mTtsSpeaker;
private CameraHandler mCameraHandler;
private TensorFlowImageClassifier mTensorFlowClassifier;
private HandlerThread mBackgroundThread;
private Handler mBackgroundHandler;
private ImageView mImage;
private TextView mResultText;
private AtomicBoolean mReady = new AtomicBoolean(false);
private ButtonInputDriver mButtonDriver;
private Gpio mReadyLED;
@Override
protected void onCreate(final Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
setContentView(R.layout.activity_camera);
mImage = findViewById(R.id.imageView);
mResultText = findViewById(R.id.resultText);
init();
CameraHandler.dumpFormatInfo(this);
}
private void init() {
if (isAndroidThingsDevice(this)) {
initPIO();
}
mBackgroundThread = new HandlerThread("BackgroundThread");
mBackgroundThread.start();
mBackgroundHandler = new Handler(mBackgroundThread.getLooper());
mBackgroundHandler.post(mInitializeOnBackground);
}
/**
* This method should only be called when running on an Android Things device.
*/
private void initPIO() {
PeripheralManager pioManager = PeripheralManager.getInstance();
try {
mReadyLED = pioManager.openGpio(BoardDefaults.getGPIOForLED());
mReadyLED.setDirection(Gpio.DIRECTION_OUT_INITIALLY_LOW);
mButtonDriver = new ButtonInputDriver(
BoardDefaults.getGPIOForButton(),
Button.LogicState.PRESSED_WHEN_LOW,
SHUTTER_KEYCODE);
mButtonDriver.register();
} catch (IOException e) {
mButtonDriver = null;
Log.w(TAG, "Could not open GPIO pins", e);
}
}
private Runnable mInitializeOnBackground = new Runnable() {
@Override
public void run() {
mCameraHandler = CameraHandler.getInstance();
try {
mCameraHandler.initializeCamera(ImageClassifierActivity.this,
mBackgroundHandler, MODEL_IMAGE_SIZE, ImageClassifierActivity.this);
CameraHandler.dumpFormatInfo(ImageClassifierActivity.this);
} catch (CameraAccessException e) {
throw new RuntimeException(e);
}
Size cameraCaptureSize = mCameraHandler.getImageDimensions();
mImagePreprocessor =
new ImagePreprocessor(cameraCaptureSize.getWidth(), cameraCaptureSize.getHeight(),
MODEL_IMAGE_SIZE.getWidth(), MODEL_IMAGE_SIZE.getHeight());
mTtsSpeaker = new TtsSpeaker();
mTtsSpeaker.setHasSenseOfHumor(true);
mTtsEngine = new TextToSpeech(ImageClassifierActivity.this,
new TextToSpeech.OnInitListener() {
@Override
public void onInit(int status) {
if (status == TextToSpeech.SUCCESS) {
mTtsEngine.setLanguage(Locale.US);
mTtsEngine.setOnUtteranceProgressListener(utteranceListener);
mTtsSpeaker.speakReady(mTtsEngine);
} else {
Log.w(TAG, "Could not open TTS Engine (onInit status=" + status
+ "). Ignoring text to speech");
mTtsEngine = null;
}
}
});
try {
mTensorFlowClassifier = new TensorFlowImageClassifier(ImageClassifierActivity.this,
MODEL_IMAGE_SIZE.getWidth(), MODEL_IMAGE_SIZE.getHeight());
} catch (IOException e) {
throw new IllegalStateException("Cannot initialize TFLite Classifier", e);
}
setReady(true);
}
};
private Runnable mBackgroundClickHandler = new Runnable() {
@Override
public void run() {
if (mTtsEngine != null) {
mTtsSpeaker.speakShutterSound(mTtsEngine);
}
mCameraHandler.takePicture();
}
};
private UtteranceProgressListener utteranceListener = new UtteranceProgressListener() {
@Override
public void onStart(String utteranceId) {
setReady(false);
}
@Override
public void onDone(String utteranceId) {
setReady(true);
}
@Override
public void onError(String utteranceId) {
setReady(true);
}
};
@Override
public boolean onKeyUp(int keyCode, KeyEvent event) {
Log.d(TAG, "Received key up: " + keyCode);
if (keyCode == SHUTTER_KEYCODE) {
startImageCapture();
return true;
}
return super.onKeyUp(keyCode, event);
}
/**
* Invoked when the user taps on the UI from a touch-enabled display
*/
public void onShutterClick(View view) {
Log.d(TAG, "Received screen tap");
startImageCapture();
}
/**
* Verify and initiate a new image capture
*/
private void startImageCapture() {
boolean isReady = mReady.get();
Log.d(TAG, "Ready for another capture? " + isReady);
if (isReady) {
setReady(false);
mResultText.setText("Hold on...");
mBackgroundHandler.post(mBackgroundClickHandler);
} else {
Log.i(TAG, "Sorry, processing hasn't finished. Try again in a few seconds");
}
}
/**
* Mark the system as ready for a new image capture
*/
private void setReady(boolean ready) {
mReady.set(ready);
if (mReadyLED != null) {
try {
mReadyLED.setValue(ready);
} catch (IOException e) {
Log.w(TAG, "Could not set LED", e);
}
}
}
@Override
public void onImageAvailable(ImageReader reader) {
final Bitmap bitmap;
try (Image image = reader.acquireNextImage()) {
bitmap = mImagePreprocessor.preprocessImage(image);
}
runOnUiThread(new Runnable() {
@Override
public void run() {
mImage.setImageBitmap(bitmap);
}
});
final Collection<Recognition> results = mTensorFlowClassifier.doRecognize(bitmap);
Log.d(TAG, "Got the following results from Tensorflow: " + results);
runOnUiThread(new Runnable() {
@Override
public void run() {
if (results == null || results.isEmpty()) {
mResultText.setText("I don't understand what I see");
} else {
StringBuilder sb = new StringBuilder();
Iterator<Recognition> it = results.iterator();
int counter = 0;
while (it.hasNext()) {
Recognition r = it.next();
sb.append(r.getTitle());
counter++;
if (counter < results.size() - 1 ) {
sb.append(", ");
} else if (counter == results.size() - 1) {
sb.append(" or ");
}
}
mResultText.setText(sb.toString());
}
}
});
if (mTtsEngine != null) {
// speak out loud the result of the image recognition
mTtsSpeaker.speakResults(mTtsEngine, results);
} else {
// if theres no TTS, we don't need to wait until the utterance is spoken, so we set
// to ready right away.
setReady(true);
}
}
@Override
protected void onDestroy() {
super.onDestroy();
try {
if (mBackgroundThread != null) mBackgroundThread.quit();
} catch (Throwable t) {
// close quietly
}
mBackgroundThread = null;
mBackgroundHandler = null;
try {
if (mCameraHandler != null) mCameraHandler.shutDown();
} catch (Throwable t) {
// close quietly
}
try {
if (mTensorFlowClassifier != null) mTensorFlowClassifier.destroyClassifier();
} catch (Throwable t) {
// close quietly
}
try {
if (mButtonDriver != null) mButtonDriver.close();
} catch (Throwable t) {
// close quietly
}
if (mTtsEngine != null) {
mTtsEngine.stop();
mTtsEngine.shutdown();
}
}
/**
* @return true if this device is running Android Things.
*
* Source: https://stackoverflow.com/a/44171734/112705
*/
private boolean isAndroidThingsDevice(Context context) {
// We can't use PackageManager.FEATURE_EMBEDDED here as it was only added in API level 26,
// and we currently target a lower minSdkVersion
final PackageManager pm = context.getPackageManager();
boolean isRunningAndroidThings = pm.hasSystemFeature("android.hardware.type.embedded");
Log.d(TAG, "isRunningAndroidThings: " + isRunningAndroidThings);
return isRunningAndroidThings;
}
}
