javascript - 相机在使用 p5.js 的 webview android 中不起作用

Question

我想在 Android Studio 中使用 p5.js 和 ml5.js 库。我已经设法在android中使用webview加载html文件。当应用程序启动但相机为黑色时，也会创建画布。我的应用程序将通过从相机中检测姿势来对姿势进行分类。我是机器学习和这个库的新手。任何帮助，将不胜感激。谢谢。

学习.js

let video;
let poseNet;
let pose;
let skeleton;
let thirtysecs;
let posesArray = ['Mountain', 'Right Tree', 'Left Tree', 'Right March', 'Left March', 'Warrior','Right Warrior','Knee Stretch Right','Knee Stretch Left'];
var imgArray = new Array();

let dingdong;

var poseImage;

let yogi;
let poseLabel;

var targetLabel;
var errorCounter;
var iterationCounter;
var poseCounter;
var target;

var timeLeft;

function setup() {
  var canvas = createCanvas(640, 480);
  canvas.position(80, 90);
  video = createCapture(VIDEO);
  video.hide();
  dingdong = loadSound('bell.mp3');
  poseNet = ml5.poseNet(video, modelLoaded);
  poseNet.on('pose', gotPoses);

  imgArray[0] = new Image();
  imgArray[0].src = 'imgs/mountain.svg';
  imgArray[1] = new Image();
  imgArray[1].src = 'imgs/tree.svg';
  imgArray[2] = new Image();
  imgArray[2].src = 'imgs/mountain.svg';
  imgArray[3] = new Image();
  imgArray[3].src = 'imgs/mountain.svg';
  imgArray[4] = new Image();
  imgArray[4].src = 'imgs/tree.svg';
  imgArray[5] = new Image();
  imgArray[5].src = 'imgs/mountain.svg';
  imgArray[6] = new Image();
  imgArray[6].src = 'imgs/tree.svg';
  imgArray[7] = new Image();
  imgArray[7].src = 'imgs/mountain.svg';
  imgArray[8] = new Image();
  imgArray[8].src = 'imgs/tree.svg';

  
  poseCounter = 0;
  targetLabel = 1;
  target = posesArray[poseCounter];
  document.getElementById("poseName").textContent = target;
  timeLeft = 10;
  document.getElementById("time").textContent = "00:" + timeLeft;
  errorCounter = 0;
  iterationCounter = 0;
  document.getElementById("poseImg").src = imgArray[poseCounter].src;
  
  let options = {
    inputs: 50,
    outputs: 15,
    task: 'classification',
    debug: true
  }
  
  yogi = ml5.neuralNetwork(options);
  const modelInfo = {
    model: './model/model.json',
    metadata: './model/model_meta.json',
    weights: './model/model.weights.bin',
  };
  yogi.load(modelInfo, yogiLoaded);
}
  
function yogiLoaded(){
  console.log("Model ready!");
  classifyPose();
}

function classifyPose(){
  if (pose) {
    let inputs = [];
    for (let i = 0; i < pose.keypoints.length; i++) {
      let x = pose.keypoints[i].position.x;
      let y = pose.keypoints[i].position.y;
      inputs.push(x);
      inputs.push(y);
    }
    yogi.classify(inputs, gotResult);
  } else {
    console.log("Pose not found");
    setTimeout(classifyPose, 100);
  }
}

function gotResult(error, results) {
  document.getElementById("welldone").textContent = "";
  document.getElementById("sparkles").style.display = "none";
  if (results[0].confidence > 0.75) 
  {
    dingdong.play();
    console.log("Confidence");
    if (results[0].label == targetLabel.toString()){
      console.log(targetLabel);
      iterationCounter = iterationCounter + 1;

      console.log(iterationCounter)
      
      if (iterationCounter == 10) {
        console.log("30!")
        iterationCounter = 0;
        nextPose();}
      else{
        console.log("doin this")
        timeLeft = timeLeft - 1;
        if (timeLeft < 10){
          document.getElementById("time").textContent = "00:0" + timeLeft;
        }else{
        document.getElementById("time").textContent = "00:" + timeLeft;}
        setTimeout(classifyPose, 1000);}}
    else{
      errorCounter = errorCounter + 1;
      console.log("error");
      if (errorCounter >= 4){
        console.log("four errors");
        iterationCounter = 0;
        timeLeft = 10;
        if (timeLeft < 10){
          document.getElementById("time").textContent = "00:0" + timeLeft;
    
        }else{
        document.getElementById("time").textContent = "00:" + timeLeft;}
        errorCounter = 0;
        dingdong.play();
        setTimeout(classifyPose, 100);
      }else{
        setTimeout(classifyPose, 100);
      }}}
  else{
    console.log("whatwe really dont want")
    setTimeout(classifyPose, 100);
}}


function gotPoses(poses) {
  if (poses.length > 0) {
    pose = poses[0].pose;
    skeleton = poses[0].skeleton;
  }
}

function modelLoaded() {
  document.getElementById("rectangle").style.display = "none";
  console.log('poseNet ready');
}

function draw() {
  push();
  translate(video.width, 0);
  scale(-1,1);
  image(video, 0, 0, video.width, video.height);
  
  if (pose) {
    for (let i = 0; i < skeleton.length; i++) {
      let a = skeleton[i][0];
      let b = skeleton[i][1];
      strokeWeight(8);
      stroke(244, 194, 194);
      line(a.position.x, a.position.y, b.position.x, b.position.y);
    }
  }
  pop();
}

function nextPose(){
  if (poseCounter >= 8) {
    console.log("Well done, you have learnt all poses!");
    document.getElementById("finish").textContent = "Amazing!";
    document.getElementById("welldone").textContent = "All poses done.";
    document.getElementById("sparkles").style.display = 'block';
  }else{
    console.log("Well done, you all poses!");
    errorCounter = 0;
    iterationCounter = 0;
    poseCounter = poseCounter + 1;
    targetLabel = poseCounter + 1;
    console.log("next pose target label" + targetLabel)
    target = posesArray[poseCounter];
    document.getElementById("poseName").textContent = target;
    document.getElementById("welldone").textContent = "Well done, next pose!";
    document.getElementById("sparkles").style.display = 'block';
    document.getElementById("poseImg").src = imgArray[poseCounter].src;
    console.log("classifying again");
    timeLeft = 10;
    document.getElementById("time").textContent = "00:" + timeLeft;
    setTimeout(classifyPose, 4000)}
}

MainActivity.java

import android.Manifest;
import android.annotation.TargetApi;
import android.content.pm.PackageManager;
import android.os.Build;
import android.os.Bundle;
import android.util.Log;
import android.webkit.PermissionRequest;
import android.webkit.WebChromeClient;
import android.webkit.WebView;
import android.webkit.WebViewClient;

import androidx.appcompat.app.AppCompatActivity;

import java.util.ArrayList;
import java.util.List;

public class MainActivity extends AppCompatActivity {

    private WebView webview;
    String TAG="Debug";

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        webview =(WebView)findViewById(R.id.webView);

        webview.setWebViewClient(new WebViewClient());
        webview.getSettings().setJavaScriptEnabled(true);
        webview.getSettings().setDomStorageEnabled(true);
        webview.getSettings().setAllowFileAccessFromFileURLs(true);
        webview.getSettings().setAllowUniversalAccessFromFileURLs(true);
        webview.setOverScrollMode(WebView.OVER_SCROLL_NEVER);

        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
            int hasCameraPermission = checkSelfPermission(Manifest.permission.CAMERA);
            Log.d(TAG, "has camera permission: " + hasCameraPermission);
            int hasRecordPermission = checkSelfPermission(Manifest.permission.RECORD_AUDIO);
            Log.d(TAG, "has record permission: " + hasRecordPermission);
            int hasAudioPermission = checkSelfPermission(Manifest.permission.MODIFY_AUDIO_SETTINGS);
            Log.d(TAG, "has audio permission: " + hasAudioPermission);
            List<String> permissions = new ArrayList<>();
            if (hasCameraPermission != PackageManager.PERMISSION_GRANTED) {
                permissions.add(Manifest.permission.CAMERA);
            }
            if (hasRecordPermission != PackageManager.PERMISSION_GRANTED) {
                permissions.add(Manifest.permission.RECORD_AUDIO);
            }
            if (hasAudioPermission != PackageManager.PERMISSION_GRANTED) {
                permissions.add(Manifest.permission.MODIFY_AUDIO_SETTINGS);
            }

            if (!permissions.isEmpty()) {
                requestPermissions(permissions.toArray(new String[permissions.size()]),111);

            }
        }

        webview.setWebChromeClient(new WebChromeClient() {
            @TargetApi(Build.VERSION_CODES.LOLLIPOP)
            @Override
            public void onPermissionRequest(final PermissionRequest request) {
                MainActivity.this.runOnUiThread(new Runnable() {
                    @Override
                    public void run() {
                        request.grant(request.getResources());
                    }
                });
            }

            @Override
            public void onPermissionRequestCanceled(PermissionRequest request) {
                Log.d(TAG, "onPermissionRequestCanceled");
            }
        });

        webview.loadUrl("file:///android_asset/learn.html");

    }
}

学习.html

<!DOCTYPE html>
<html>
    <head>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
      
        <title>HappyFit</title>
      
        <link rel="icon" type="image/png" sizes="32x32" href="imgs/yoga 1.png">
      
        <link rel="stylesheet" href="css/lpmain.css">
      
        <!-- Google Fonts call. Font Used Raleway -->
        <link href="http://fonts.googleapis.com/css?family=Raleway:400,300" rel="stylesheet" type="text/css">
      
    </head>

    <body>
        <img src="imgs/blob3.svg" id="blob" style="position:absolute; width: 500px; height: 450px; left: -210px; top: -290px;">
        <h1 style="top:-30px">Learn</h1>
        <div id="rectangle">Loading...</div>
        <img src="imgs/sparkles.svg" id="sparkles" style="top:65px; left:800px; position:absolute; width:120px; height:107px; z-index:1; display:none">
        <h1 id="finish" style="font-size:40px; top:-20px; left:710px; position:absolute;"></h1>
        <h1 id="welldone" style="font-size:40px; top:35px; left:810px; position:absolute;"></h1>
        <div class="parent">
        <h1 id="poseName" class="child"><span>Tree</span></h1>
        </div>
        <img id="poseImg" src="imgs/mountain.svg">
        
        <h1 style="font-size: 55px; top:480px; left:850px"><span id="time">00:10</span></h1>
        <h1 style="font-size: 40px; top:540px; left:750px">seconds left to hold</h1>
        <div><a href="index.html" class="btns">BACK</a></div>


        <script src="src/p5.min.js"></script>
        <script src="src/p5.sound.min.js"></script>
        <script src="https://unpkg.com/ml5@0.5.0/dist/ml5.min.js"></script>
        <script src="learn.js"></script>    
        
        <footer>&copy; HappyFit</footer>
    </body>
</html>

javascript - 相机在使用 p5.js 的 webview android 中不起作用

0 回答 0

Related

Reference