1

基本上我想做的是在视频会议中本地检测人脸。为此，我将RTCMultiConnection用于视频会议部分，确切地说是这个特定部分，并将 OpenCV.js 用于计算机视觉部分，特别是这个存储库。

我知道我需要将 mediaStream 对象从视频元素发送到 OpenCV 函数内的画布元素。我没有在画布上显示该流，所以我想我会将包含该 mediaStream 对象的变量设为全局，以便我可以将它传递给 OpenCV 函数，但这也不起作用。

如果您能提供任何见解，我将不胜感激，谢谢。

这是代码：

<!DOCTYPE html>
<html lang="en" dir="ltr">
<head>
  <meta charset="utf-8">
  <title>Video Conferencing using RTCMultiConnection</title>
  <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0">
  <link rel="shortcut icon" href="/demos/logo.png">
  <link rel="stylesheet" href="/demos/stylesheet.css">
  <script src="/demos/menu.js"></script>
  <script async src="js/opencv.js" ></script>
  <script src="js/utils.js"></script>
</head>
<body>
  <header>
    <!-- <a class="logo" href="/"><img src="/demos/logo.png" alt="RTCMultiConnection"></a>
    <a href="/" class="menu-explorer">Menu<img src="/demos/menu-icon.png" alt="Menu"></a>
    <!-- <nav>
      <li>
        <a href="/">Home</a>
      </li>
      <li>
        <a href="/demos/">Demos</a>
      </li>
      <li>
        <a href="https://www.rtcmulticonnection.org/docs/getting-started/">Getting Started</a>
      </li>
      <li>
        <a href="https://www.rtcmulticonnection.org/FAQ/">FAQ</a>
      </li>
      <li>
        <a href="https://www.youtube.com/playlist?list=PLPRQUXAnRydKdyun-vjKPMrySoow2N4tl">YouTube</a>
      </li>
      <li>
        <a href="https://github.com/muaz-khan/RTCMultiConnection/wiki">Wiki</a>
      </li>
      <li>
        <a href="https://github.com/muaz-khan/RTCMultiConnection">Github</a>
      </li>
    </nav> -->
  </header> -->

  <h1>
    Video Conferencing using RTCMultiConnection
    <p class="no-mobile">
      Multi-user (many-to-many) video chat using mesh networking model.
    </p>
  </h1>

  <section class="make-center">
    <div>
      <label><input type="checkbox" id="record-entire-conference"> Record Entire Conference In The Browser?</label>
      <span id="recording-status" style="display: none;"></span>
      <button id="btn-stop-recording" style="display: none;">Stop Recording</button>
      <br><br>

      <input type="text" id="room-id" value="abcdef" autocorrect=off autocapitalize=off size=20>
      <button id="open-room">Open Room</button>
      <button id="join-room">Join Room</button>
      <button id="open-or-join-room">Auto Open Or Join Room</button>
    </div>

    <div id="videos-container" style="margin: 20px 0;"></div>
    <canvas id="canvas_output"></canvas>

    <div id="room-urls" style="text-align: center;display: none;background: #F1EDED;margin: 15px -10px;border: 1px solid rgb(189, 189, 189);border-left: 0;border-right: 0;"></div>
  </section>

<script src="/dist/RTCMultiConnection.min.js"></script>
<script src="/node_modules/webrtc-adapter/out/adapter.js"></script>
<script src="/socket.io/socket.io.js"></script>
<!-- <script async src="js/opencv.js" onload="openCvReady()"></script> -->
<!-- <script src="js/utils.js"></script> -->

<!-- custom layout for HTML5 audio/video elements -->
<link rel="stylesheet" href="/dev/getHTMLMediaElement.css">
<script src="/dev/getHTMLMediaElement.js"></script>

<script src="/node_modules/recordrtc/RecordRTC.js"></script>
<script>
// ......................................................
// .......................UI Code........................
// ......................................................
document.getElementById('open-room').onclick = function() {
    disableInputButtons();
    connection.open(document.getElementById('room-id').value, function(isRoomOpened, roomid, error) {
        if(isRoomOpened === true) {
          showRoomURL(connection.sessionid);
        }
        else {
          disableInputButtons(true);
          if(error === 'Room not available') {
            alert('Someone already created this room. Please either join or create a separate room.');
            return;
          }
          alert(error);
        }
    });
};

document.getElementById('join-room').onclick = function() {
    disableInputButtons();
    connection.join(document.getElementById('room-id').value, function(isJoinedRoom, roomid, error) {
      if (error) {
            disableInputButtons(true);
            if(error === 'Room not available') {
              alert('This room does not exist. Please either create it or wait for moderator to enter in the room.');
              return;
            }
            alert(error);
        }
    });
};

document.getElementById('open-or-join-room').onclick = function() {
    disableInputButtons();
    connection.openOrJoin(document.getElementById('room-id').value, function(isRoomExist, roomid, error) {
        if(error) {
          disableInputButtons(true);
          alert(error);
        }
        else if (connection.isInitiator === true) {
            // if room doesn't exist, it means that current user will create the room
            showRoomURL(roomid);
        }
    });
};
// ......................................................
// .......................UI Code ENDS........................
// ......................................................



// ***********************************************************




// ......................................................
// ..................RTCMultiConnection Code.............
// ......................................................

var connection = new RTCMultiConnection();

// by default, socket.io server is assumed to be deployed on your own URL
connection.socketURL = '/';

// comment-out below line if you do not have your own socket.io server
// connection.socketURL = 'https://rtcmulticonnection.herokuapp.com:443/';

connection.socketMessageEvent = 'video-conference-demo';

connection.session = {
    audio: true,
    video: true
};

connection.sdpConstraints.mandatory = {
    OfferToReceiveAudio: true,
    OfferToReceiveVideo: true
};

// STAR_FIX_VIDEO_AUTO_PAUSE_ISSUES
// via: https://github.com/muaz-khan/RTCMultiConnection/issues/778#issuecomment-524853468
var bitrates = 512;
var resolutions = 'Ultra-HD';
var videoConstraints = {};

if (resolutions == 'HD') {
    videoConstraints = {
        width: {
            ideal: 1280
        },
        height: {
            ideal: 720
        },
        frameRate: 30
    };
}

if (resolutions == 'Ultra-HD') {
    videoConstraints = {
        width: {
            ideal: 1920
        },
        height: {
            ideal: 1080
        },
        frameRate: 30
    };
}

connection.mediaConstraints = {
    video: videoConstraints,
    audio: true
};

var CodecsHandler = connection.CodecsHandler;

connection.processSdp = function(sdp) {
    var codecs = 'vp8';
    
    if (codecs.length) {
        sdp = CodecsHandler.preferCodec(sdp, codecs.toLowerCase());
    }

    if (resolutions == 'HD') {
        sdp = CodecsHandler.setApplicationSpecificBandwidth(sdp, {
            audio: 128,
            video: bitrates,
            screen: bitrates
        });

        sdp = CodecsHandler.setVideoBitrates(sdp, {
            min: bitrates * 8 * 1024,
            max: bitrates * 8 * 1024,
        });
    }

    if (resolutions == 'Ultra-HD') {
        sdp = CodecsHandler.setApplicationSpecificBandwidth(sdp, {
            audio: 128,
            video: bitrates,
            screen: bitrates
        });

        sdp = CodecsHandler.setVideoBitrates(sdp, {
            min: bitrates * 8 * 1024,
            max: bitrates * 8 * 1024,
        });
    }

    return sdp;
};
// END_FIX_VIDEO_AUTO_PAUSE_ISSUES

// https://www.rtcmulticonnection.org/docs/iceServers/
// use your own TURN-server here!
connection.iceServers = [{
    'urls': [
        'stun:stun.l.google.com:19302',
        'stun:stun1.l.google.com:19302',
        'stun:stun2.l.google.com:19302',
        'stun:stun.l.google.com:19302?transport=udp',
    ]
}];

connection.videosContainer = document.getElementById('videos-container');

connection.onstream = function(specialEvent) {
  // its just event but I was debugging it on the console and wanted to change the 
  // name to specialEvent
    var existing = document.getElementById(specialEvent.streamid);
    if(existing && existing.parentNode) {
      existing.parentNode.removeChild(existing);
    }

    // specialEvent.mediaElement.removeAttribute('src');
    // specialEvent.mediaElement.removeAttribute('srcObject');
    specialEvent.mediaElement.muted = true;
    specialEvent.mediaElement.volume = 0;

    var video = document.createElement('video');
    video.setAttribute("id", "cam_input");
    window.video = video; 

    try {
        video.setAttributeNode(document.createAttribute('autoplay'));
        video.setAttributeNode(document.createAttribute('playsinline'));
    } catch (e) {
        video.setAttribute('autoplay', true);
        video.setAttribute('playsinline', true);
    }

    if(specialEvent.type === 'local') {
      video.volume = 0;
      try {
          video.setAttributeNode(document.createAttribute('muted'));
      } catch (e) {
          video.setAttribute('muted', true);
      }
    }
    video.srcObject = specialEvent.stream;

    var width = parseInt(connection.videosContainer.clientWidth / 3) - 20;
    var mediaElement = getHTMLMediaElement(video, {
        title: specialEvent.userid,
        buttons: ['full-screen'],
        width: width,
        showOnMouseEnter: false
    });

    connection.videosContainer.appendChild(mediaElement);

    setTimeout(function() {
        mediaElement.media.play();
    }, 5000);

    mediaElement.id = specialEvent.streamid;

    // to keep room-id in cache
    localStorage.setItem(connection.socketMessageEvent, connection.sessionid);

    chkRecordConference.parentNode.style.display = 'none';

    if(chkRecordConference.checked === true) {
      btnStopRecording.style.display = 'inline-block';
      recordingStatus.style.display = 'inline-block';

      var recorder = connection.recorder;
      if(!recorder) {
        recorder = RecordRTC([specialEvent.stream], {
          type: 'video'
        });
        recorder.startRecording();
        connection.recorder = recorder;
      }
      else {
        recorder.getInternalRecorder().addStreams([specialEvent.stream]);
      }

      if(!connection.recorder.streams) {
        connection.recorder.streams = [];
      }

      connection.recorder.streams.push(specialEvent.stream);
      recordingStatus.innerHTML = 'Recording ' + connection.recorder.streams.length + ' streams';
    }

    if(specialEvent.type === 'local') {
      connection.socket.on('disconnect', function() {
        if(!connection.getAllParticipants().length) {
          location.reload();
        }
      });
    }

  window.specialEvent = specialEvent; 
};



// ******************************
// 
// OPENCV CODE STARTS FROM HERE
// 
// ******************************

function openCvReady() {
  // cv['onRuntimeInitialized']=()=>{
    // let video = document.getElementById("cam_input"); // video is the id of video tag
    // navigator.mediaDevices.getUserMedia({ video: true, audio: false })
    // .then(function(stream) {
    //     console.log(stream);
    //     video.srcObject = stream;
    //     video.play();
    // })
    // .catch(function(err) {
    //     console.log("An error occurred! " + err);
    // });
    // let cam_input = document.getElementById("cam_input"); 
    let video = document.getElementById("cam_input"); 
    let src = new cv.Mat(video.height, video.width, cv.CV_8UC4);
    let dst = new cv.Mat(video.height, video.width, cv.CV_8UC1);
    let gray = new cv.Mat();
    let cap = new cv.VideoCapture(cam_input);
    let faces = new cv.RectVector();
    // if (face == true){
    //     console.log("face shown");
    // }
    let classifier = new cv.CascadeClassifier();
    let utils = new Utils('errorMessage');
    
    let faceCascadeFile = 'haarcascade_frontalface_default.xml'; // path to xml
    utils.createFileFromUrl(faceCascadeFile, faceCascadeFile, () => {
    classifier.load(faceCascadeFile); // in the callback, load the cascade from file 
});
    const FPS = 24;
    function processVideo() {
        let begin = Date.now();
        cap.read(src);
        src.copyTo(dst);
        cv.cvtColor(dst, gray, cv.COLOR_RGBA2GRAY, 0);
        try{
            classifier.detectMultiScale(gray, faces, 1.1, 3, 0);
            console.log(faces.size());
        }catch(err){
            console.log(err);
        }
        for (let i = 0; i < faces.size(); ++i) {
            let face = faces.get(i);
            console.log("face:");
            console.log(face);
            let point1 = new cv.Point(face.x, face.y);
            let point2 = new cv.Point(face.x + face.width, face.y + face.height);
            cv.rectangle(dst, point1, point2, [255, 0, 0, 255]);
        }
        cv.imshow("canvas_output", dst);
        // schedule next one.
        let delay = 1000/FPS - (Date.now() - begin);
        setTimeout(processVideo, delay);
}
// schedule first one.
setTimeout(processVideo, 0);
  // };
}

var recordingStatus = document.getElementById('recording-status');
var chkRecordConference = document.getElementById('record-entire-conference');
var btnStopRecording = document.getElementById('btn-stop-recording');
btnStopRecording.onclick = function() {
  var recorder = connection.recorder;
  if(!recorder) return alert('No recorder found.');
  recorder.stopRecording(function() {
    var blob = recorder.getBlob();
    invokeSaveAsDialog(blob);

    connection.recorder = null;
    btnStopRecording.style.display = 'none';
    recordingStatus.style.display = 'none';
    chkRecordConference.parentNode.style.display = 'inline-block';
  });
};

connection.onstreamended = function(event) {
    var mediaElement = document.getElementById(event.streamid);
    if (mediaElement) {
        mediaElement.parentNode.removeChild(mediaElement);
    }
};

connection.onMediaError = function(e) {
    if (e.message === 'Concurrent mic process limit.') {
        if (DetectRTC.audioInputDevices.length <= 1) {
            alert('Please select external microphone. Check github issue number 483.');
            return;
        }

        var secondaryMic = DetectRTC.audioInputDevices[1].deviceId;
        connection.mediaConstraints.audio = {
            deviceId: secondaryMic
        };

        connection.join(connection.sessionid);
    }
};

// ..................................
// ALL below scripts are redundant!!!
// ..................................

function disableInputButtons(enable) {
    document.getElementById('room-id').onkeyup();

    document.getElementById('open-or-join-room').disabled = !enable;
    document.getElementById('open-room').disabled = !enable;
    document.getElementById('join-room').disabled = !enable;
    document.getElementById('room-id').disabled = !enable;
}

// ......................................................
// ......................Handling Room-ID................
// ......................................................

function showRoomURL(roomid) {
    var roomHashURL = '#' + roomid;
    var roomQueryStringURL = '?roomid=' + roomid;

    var html = '<h2>Unique URL for your room:</h2><br>';

    html += 'Hash URL: <a href="' + roomHashURL + '" target="_blank">' + roomHashURL + '</a>';
    html += '<br>';
    html += 'QueryString URL: <a href="' + roomQueryStringURL + '" target="_blank">' + roomQueryStringURL + '</a>';

    var roomURLsDiv = document.getElementById('room-urls');
    roomURLsDiv.innerHTML = html;

    roomURLsDiv.style.display = 'block';
}

(function() {
    var params = {},
        r = /([^&=]+)=?([^&]*)/g;

    function d(s) {
        return decodeURIComponent(s.replace(/\+/g, ' '));
    }
    var match, search = window.location.search;
    while (match = r.exec(search.substring(1)))
        params[d(match[1])] = d(match[2]);
    window.params = params;
})();

var roomid = '';
if (localStorage.getItem(connection.socketMessageEvent)) {
    roomid = localStorage.getItem(connection.socketMessageEvent);
} else {
    roomid = connection.token();
}

var txtRoomId = document.getElementById('room-id');
txtRoomId.value = roomid;
txtRoomId.onkeyup = txtRoomId.oninput = txtRoomId.onpaste = function() {
    localStorage.setItem(connection.socketMessageEvent, document.getElementById('room-id').value);
};

var hashString = location.hash.replace('#', '');
if (hashString.length && hashString.indexOf('comment-') == 0) {
    hashString = '';
}

var roomid = params.roomid;
if (!roomid && hashString.length) {
    roomid = hashString;
}

if (roomid && roomid.length) {
    document.getElementById('room-id').value = roomid;
    localStorage.setItem(connection.socketMessageEvent, roomid);

    // auto-join-room
    (function reCheckRoomPresence() {
        connection.checkPresence(roomid, function(isRoomExist) {
            if (isRoomExist) {
                connection.join(roomid);
                return;
            }

            setTimeout(reCheckRoomPresence, 5000);
        });
    })();

    disableInputButtons();
}

// detect 2G
if(navigator.connection &&
   navigator.connection.type === 'cellular' &&
   navigator.connection.downlinkMax <= 0.115) {
  alert('2G is not supported. Please use a better internet service.');
}
</script>

  <footer>
    <small id="send-message"></small>
  </footer>

  <script src="https://www.webrtc-experiment.com/common.js"></script>
</body>
</html>
4

0 回答 0