How to do image processing in Android?
My software is used in this poster
Android Canvas
This section is the design one function in Android: pick up a photo from gallery and desplay in the App.
Tool
I used Android Studio to develop my app. It is a mature technology that helps me to build app easily.
Add ImageView and Button
In the “activity_main.xml” add a Button like this:
<Button
android:id="@+id/open_picture"
style="@style/Base.Widget.AppCompat.Button.Colored"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:text="@string/open_picture" />
Also add the ImageView:
<ImageView
android:id="@+id/imageView"
android:layout_width="match_parent"
android:layout_height="500dp"
android:scaleType="fitCenter"
android:src="@drawable/sample" />
Implement the interface logic and Pick Up Logic
Go to Java directory find the “MainActivity” class to implement the Logic First we need define two different code indicate the if the app was granted to open the camera.
private static final int IMAGE_PICK_CODE = 1000;
private static final int PERMISSION_CODE = 1001;
We need define a button object called open button and get the UI information on the button object. Use the “findViewById” function to get the object.
openBtn = findViewById(R.id.open_picture);
After we get the button object, we need add a OnClickListener(). We need check the permission code if the permission was granted to the app.
openBtn.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
//check runtime permission
if(Build.VERSION.SDK_INT >= Build.VERSION_CODES.M){
if(checkSelfPermission((Manifest.permission.READ_EXTERNAL_STORAGE))
== PackageManager.PERMISSION_DENIED){
//permission not granted request
String[] permissions = {Manifest.permission.READ_EXTERNAL_STORAGE};
//show popup for runtime permission
requestPermissions(permissions, PERMISSION_CODE);
}
else{
//permission already granted
pickImageFromGallery();
}
}
else{
//system os is less then marshmallow
pickImageFromGallery();
}
}
});
Notice there is a function called pickImageFromGallery() which is the method that actually pick the image in the gallery. We need create a Intent which is ACTION_PICK intent and set the select type to all the image file. Pass the intent and IMAGE_PICK_CODE into the startActivityForResult() function.
private void pickImageFromGallery(){
//intent to pick image
Intent intent = new Intent((Intent.ACTION_PICK));
intent.setType("image/*");
startActivityForResult(intent, IMAGE_PICK_CODE);
}
In the first time to open gallery, we need to ask the user if grant the permission to open the gallery.
@Override
public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
switch (requestCode){
case PERMISSION_CODE: {
if(grantResults.length>0 && grantResults[0] == PackageManager.PERMISSION_GRANTED){
// permission granted
pickImageFromGallery();
}
else{
// permission denied
Toast.makeText(this,"Permission denied!",Toast.LENGTH_SHORT).show();
}
}
}
}
When the Intent get result and the request is the pick up code. We can set the images URI to the image view to display the image on the screen.
@Override
protected void onActivityResult(int requestCode, int resultCode, Intent data) {
if(resultCode == RESULT_OK && requestCode == IMAGE_PICK_CODE){
imageView.setImageURI(data.getData());
}
}
The last step is to make sure the cell phone not denied you the first add code to the Manifest file.
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
Import OpenCV to Android
Download OpenCV SDK
Import Module
File --> New -> Import Module
Import the java fold
Compare the configure information
Make sure the minSdkVersion and targetSdkVersion matches
Add dependency
File --> Project Structure --> App --> Add dependency --> OpenCV
Copy library file to App folder
Copy files in this directory
Create a folder named “jniLibs” and paste the files under the directory
Use the OpenCV library
ostu.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
OpenCVLoader.initDebug();
Bitmap originalBitmap = BitmapFactory.decodeResource(getResources(),R.drawable.sample);
Mat resultMat = new Mat();
Bitmap ouputBitmap = Bitmap.createBitmap(originalBitmap.getWidth(),originalBitmap.getHeight(),Bitmap.Config.RGB_565);
Utils.bitmapToMat(originalBitmap,resultMat);
Imgproc.cvtColor(resultMat,resultMat,Imgproc.COLOR_RGBA2GRAY,0);
Imgproc.threshold(resultMat,resultMat,0,255, Imgproc.THRESH_OTSU);
Utils.matToBitmap(resultMat,ouputBitmap);
imageView.setImageBitmap(ouputBitmap);
}
});
Available Android OpenCV Functions
Image Gradient
private void imageGradient(){
Mat grayMat = new Mat();
Mat sobel = new Mat();
Mat grad_x = new Mat();
Mat abs_grad_x = new Mat();
Mat grad_y = new Mat();
Mat abs_grad_y = new Mat();
Bitmap bitmap = BitmapFactory.decodeResource(getResources(),images[current_image]);
Mat originalMat = new Mat();
Utils.bitmapToMat(bitmap,originalMat);
Imgproc.cvtColor(originalMat, grayMat,Imgproc.COLOR_RGBA2GRAY);
Imgproc.Sobel(grayMat,grad_x,CvType.CV_16S,1,0,3,1,0);
Imgproc.Sobel(grayMat,grad_y,CvType.CV_16S,0,1,3,1,0);
Core.convertScaleAbs(grad_x,abs_grad_x);
Core.convertScaleAbs(grad_y,abs_grad_y);
Core.addWeighted(abs_grad_x,0.5,abs_grad_y,0.5,1,sobel);
Bitmap currentBitmap = Bitmap.createBitmap(bitmap.getWidth(),bitmap.getHeight(),Bitmap.Config.RGB_565);
Utils.matToBitmap(sobel,currentBitmap);
imageView.setImageBitmap(currentBitmap);
}
Canny Contours
private void cannyContours(){
Bitmap originalbitmap = BitmapFactory.decodeResource(getResources(),images[current_image]);
Mat originalMat = new Mat();
Utils.bitmapToMat(originalbitmap,originalMat);
Mat grayMat = new Mat();
Mat cannyEdges = new Mat();
Mat hierarchy = new Mat();
List<MatOfPoint> contourList = new ArrayList<MatOfPoint>();
Imgproc.cvtColor(originalMat,grayMat,Imgproc.COLOR_RGBA2GRAY);
Imgproc.Canny(grayMat,cannyEdges,10,100);
Imgproc.findContours(cannyEdges,contourList,hierarchy,Imgproc.RETR_LIST,Imgproc.CHAIN_APPROX_SIMPLE);
Mat contours = new Mat();
contours.create(cannyEdges.rows(),cannyEdges.cols(),CvType.CV_8UC3);
Random r = new Random();
for (int i = 0; i< contourList.size();i++){
Imgproc.drawContours(contours,contourList,i,new Scalar(r.nextInt(255),r.nextInt(255),r.nextInt(255)),-1);
}
Bitmap currentBitmap = Bitmap.createBitmap(originalbitmap.getWidth(),originalbitmap.getHeight(),Bitmap.Config.RGB_565);
Utils.matToBitmap(contours,currentBitmap);
imageView.setImageBitmap(currentBitmap);
}
Blur
private void blur(){
Bitmap originalbitmap = BitmapFactory.decodeResource(getResources(),images[current_image]);
Mat mat = new Mat();
Utils.bitmapToMat(originalbitmap,mat);
Imgproc.blur(mat,mat,new Size(10,10));
Utils.matToBitmap(mat,originalbitmap);
imageView.setImageBitmap(originalbitmap);
}
Median Blur
private void medianBlur(){
Bitmap originalbitmap = BitmapFactory.decodeResource(getResources(),images[current_image]);
Mat mat = new Mat();
Utils.bitmapToMat(originalbitmap,mat);
Imgproc.medianBlur(mat,mat,3);
Utils.matToBitmap(mat,originalbitmap);
imageView.setImageBitmap(originalbitmap);
}
Dilate
private void dilate(){
Bitmap originalbitmap = BitmapFactory.decodeResource(getResources(),images[current_image]);
Mat mat = new Mat();
Utils.bitmapToMat(originalbitmap,mat);
Imgproc.dilate(mat,mat,Imgproc.getStructuringElement(Imgproc.MORPH_RECT,new Size(3,3)));
Utils.matToBitmap(mat,originalbitmap);
imageView.setImageBitmap(originalbitmap);
}
OstuThreshold Implementation
Theory
All the Theory of Ostu Threshold is from this paper: Ostu’s paper. And the reference JavaScript page.
implementation
First, I tried to implement the basic code for threshold of a certain value for example: 100. Here is the Java Code in Android Studio
public static Bitmap converImage(Bitmap original){
Bitmap finalImage = Bitmap.createBitmap(original.getWidth(), original.getHeight(),original.getConfig());
int colorPixel;
int A,R,G,B;
int width = original.getWidth();
int height = original.getHeight();
for(int x = 0; x < width; x++ ){
for(int y = 0 ; y < height; y++){
colorPixel = original.getPixel(x,y);
A = Color.alpha(colorPixel);
R = Color.red(colorPixel);
G = Color.green(colorPixel);
B = Color.blue(colorPixel);
if(R > 100 ){
R = 255;
}
else{
R = 0;
}
G= R;
B= R;
finalImage.setPixel(x,y,Color.argb(A,R,G,B));
}
}
return finalImage;
}
Modify the fixed threshold into Ostu threshold
This is the method that applied the Ostu threshold for the colored image
public static Bitmap ostuConvert(Bitmap original){
Bitmap BWimg = Bitmap.createBitmap(original.getWidth(), original.getHeight(), original.getConfig());
int width = original.getWidth();
int height = original.getHeight();
int A, R, G, B, colorPixel;
double Wcv = 0, th = 0;
int[] tPXL = new int[256];
int[][] pxl = new int[width][height];
double Bw, Bm, Bv, Fw, Fm, Fv;
int np, ImgPix = 0, fth = 0;
// pixel check for histogram //
for (int x = 0; x < width; x++) {
for (int y = 0; y < height; y++) {
colorPixel = original.getPixel(x, y);
A = Color.alpha(colorPixel);
R = Color.red(colorPixel);
G = Color.green(colorPixel);
B = Color.blue(colorPixel);
int gray = (int) ( (0.2126 * R) + (0.7152 * G) + (0.0722 * B) ); // (int) ( (0.299 * R) + (0.587 * G) + (0.114 * B) );
pxl[x][y] = gray;
tPXL[gray] = tPXL[gray] + 1;
ImgPix = ImgPix + 1;
}
}
// ----- histo-variance ----- //
for (int t = 0; t < 256; t++){
Bw = 0; Bm = 0; Bv = 0;
Fw = 0; Fm = 0; Fv = 0;
np = 0;
if (t == 0){ // all white/foreground as t0 ----- //
Fw = 1;
for (int d = 0; d < 256; d++) { //mean
Fm = Fm + (d * tPXL[d]);
}
Fm = Fm / ImgPix;
for (int e = 0; e < 256; e++) { //variance
Fv = Fv + (Math.pow((e - Fm), 2) * tPXL[e]);
}
Fv = Fv / ImgPix;
}
else { // main thresholding
for (int d = 0; d < (t-1); d++){ // BG weight & mean + BG pixel
Bw = Bw + tPXL[d];
Bm = Bm + (d * tPXL[d]);
np = np + tPXL[d];
}
Bw = Bw / ImgPix;
Bm = Bm / np;
for (int e = 0; e < (t-1); e++) { //BG variance
Bv = Bv + (Math.pow((e - Bm), 2) * tPXL[e]);
}
Bv = Bv / np;
for (int j = t; j < 256; j++) { // FG weight & mean + BG pixel
Fw = Fw + tPXL[j];
Fm = Fm + (j * tPXL[j]);
np = ImgPix - np;
}
Fw = Fw / ImgPix;
Fm = Fm / np;
for (int k = t; k < 256; k++) { //FG variance
Fv = Fv + (Math.pow((k - Fm), 2) * tPXL[k]);
}
Fv = Fv / np;
}
// within class variance
Wcv = (Bw * Bv) + (Fw * Fv);
if (t == 0){
th = Wcv;
}
else if (Wcv < th){
th = Wcv;
fth = t;
}
}
// set binarize pixel
for (int x = 0; x < width; x++) {
for (int y = 0; y < height; y++) {
int fnpx = pxl[x][y];
colorPixel = original.getPixel(x, y);
A = Color.alpha(colorPixel);
if (fnpx > fth) { //R > fth
fnpx = 255;
BWimg.setPixel(x, y, Color.argb(A, fnpx, fnpx, fnpx));
}
else {
fnpx = 0;
BWimg.setPixel(x, y, Color.argb(A, fnpx, fnpx, fnpx));
}
}
}
return BWimg;
}
Height Calculation
The Height Calculation is just count the pixel in the height. So we can apply the method above and change a little. In the last for loop: “//set binarize pixel” we modify the method as following:
float top = Float.POSITIVE_INFINITY;
float bottom = Float.NEGATIVE_INFINITY;
// set binarize pixel
for (int x = 0; x < width; x++) {
for (int y = 0; y < height; y++) {
int fnpx = pxl[x][y];
colorPixel = original.getPixel(x, y);
A = Color.alpha(colorPixel);
if (fnpx > fth) { //R > fth
fnpx = 255;
}
else {
fnpx = 0;
if(top > y ){
top = y;
}
if(bottom < y){
bottom = y;
}
}
}
}
float[] pointsY = new float[]{ top * scaleFactor ,bottom * scaleFactor};
return pointsY;
I defined the two float variable initialed as POSITIVE_INFINITY and NEGATIVE_INFINITY. When fnpx <= fth begin to calculate the value. Find the minimun role and the maximum role. I returned the float arrary represent two height value. I should post the picture of processing but because of the privacy issue I won’t post it until this research period end.
Draw line and display pixel on the image view
After we have the value of top and bottom. I need to draw a red line on the image and display the pixel value between these two.
public static void drawHeight(ImageView imgView, Bitmap imageBitmap){
Bitmap bitmap = Bitmap.createBitmap(imgView.getWidth(), imgView.getHeight(), Bitmap.Config.ARGB_8888);
float scaleFactor = (float) bitmap.getHeight() / (float) imageBitmap.getHeight();
float[] points =customizeMethod.ostuCalculateHeight(imageBitmap, scaleFactor);
Canvas canvas = new Canvas(bitmap);
imgView.draw(canvas);
Paint paint = new Paint();
paint.setColor(Color.RED);
paint.setStrokeWidth(10);
float middle = imgView.getWidth() / 2; //x value of the image
canvas.drawLine(middle, points[0], middle, points[1], paint);
String pixelInfo = (int)(points[1] - points[0]) + "pixels";
paint.setColor(Color.GREEN);
paint.setTextSize(40);
canvas.drawText(pixelInfo,middle, (points[0] + points[1])/2, paint);
//draw line on image
imgView.setImageBitmap(bitmap);
}
I create this method to draw the height. There are two variable we need pass in: imgView and imageBitmap. ImageView refers the view you want to put picture, imageBitmap is the original image in Bitmap data structure.
Problems remain to solve
Runtime
So far, my app works for measure the height of a person in most circumstance. But the runtime is very slow because I put all the calculation in one thread. There are two option to make the calculation fast. The first one is parallel computing. The second is to use the alternative OpenCV solution, it might faster than my solution. So next post I will focus on how to do parallel computing on Android software and how to load OpenCV library and apply the method.
Accurate of measurement
For my application, there still some in accurate value because of the limitation of Ostu threshold method. If a person waring dark cloth and has white hair the output will be inaccurate. Therefore, to increase the accurate of the measurement. I need implement another image processing method called Image gradient. I will introduce the implementation in next blog post.
Update
After learning opencv there are a faster way to solve the ostu threshold method. The updated code using OpenCV library. Run time is less than 1s.
private void ostuThreshold(){
Bitmap bitmap = BitmapFactory.decodeResource(getResources(),images[current_image]);
Mat Rgba = new Mat();
Mat ostuMat = new Mat();
Utils.bitmapToMat(bitmap,Rgba);
Imgproc.cvtColor(Rgba,ostuMat,Imgproc.COLOR_RGBA2GRAY,0);
Imgproc.threshold(ostuMat,ostuMat,0,255,Imgproc.THRESH_OTSU);
Bitmap outputhBitmap = Bitmap.createBitmap(bitmap.getWidth(),bitmap.getHeight(),Bitmap.Config.RGB_565);
Utils.matToBitmap(ostuMat,outputhBitmap);
imageView.setImageBitmap(outputhBitmap);
}
Kmean Implementation
Understand K-mean Algorithms
K-mean clustering method originally from signal processing, this method is the popular for cluster analysis in data mining. The goal of this algorithms is to partition n observations into K clusters in which each observation belongs to the cluster with the nearest mean.
What I use K-mean for
In my projects, I need to analysis human body shape and get some health information. I need to distinguish the cloth and human body. Therefore, I need use k-mean to cluster the colors.
Code for k-mean
public void k_Mean(){
Mat rgba = new Mat();
Mat mHSV = new Mat();
Bitmap bitmap = BitmapFactory.decodeResource(getResources(),images[current_image]);
Bitmap outputBitmap = Bitmap.createBitmap(bitmap.getWidth(),bitmap.getHeight(), Bitmap.Config.RGB_565);
Utils.bitmapToMat(bitmap,rgba);
//must convert to 3 channel image
Imgproc.cvtColor(rgba, mHSV, Imgproc.COLOR_RGBA2RGB,3);
Imgproc.cvtColor(rgba, mHSV, Imgproc.COLOR_RGB2HSV,3);
Mat clusters = cluster(mHSV, 3).get(0);
Utils.matToBitmap(clusters,outputBitmap);
imageView.setImageBitmap(outputBitmap);
}
public static List<Mat> cluster(Mat cutout, int k) {
Mat samples = cutout.reshape(1, cutout.cols() * cutout.rows());
Mat samples32f = new Mat();
samples.convertTo(samples32f, CvType.CV_32F, 1.0 / 255.0);
Mat labels = new Mat();
//criteria means the maximum loop
TermCriteria criteria = new TermCriteria(TermCriteria.COUNT, 20, 1);
Mat centers = new Mat();
Core.kmeans(samples32f, k, labels, criteria, 1, Core.KMEANS_PP_CENTERS, centers);
return showClusters(cutout, labels, centers);
}
private static List<Mat> showClusters (Mat cutout, Mat labels, Mat centers) {
centers.convertTo(centers, CvType.CV_8UC1, 255.0);
centers.reshape(3);
System.out.println(labels + "labels");
List<Mat> clusters = new ArrayList<Mat>();
for(int i = 0; i < centers.rows(); i++) {
clusters.add(Mat.zeros(cutout.size(), cutout.type()));
}
Map<Integer, Integer> counts = new HashMap<Integer, Integer>();
for(int i = 0; i < centers.rows(); i++) counts.put(i, 0);
int rows = 0;
for(int y = 0; y < cutout.rows(); y++) {
for(int x = 0; x < cutout.cols(); x++) {
int label = (int)labels.get(rows, 0)[0];
int r = (int)centers.get(label, 2)[0];
int g = (int)centers.get(label, 1)[0];
int b = (int)centers.get(label, 0)[0];
counts.put(label, counts.get(label) + 1);
clusters.get(label).put(y, x, b, g, r);
rows++;
}
}
System.out.println(counts);
return clusters;
}