我已经实现了一个在 html5 上流式传输相机输出的服务。但它只有在我使用localhost :8080 localhost 时才有效，如果我使用 IP 地址或机器名，那么它甚至不会检测到摄像头。
/*global logger*/
/*
VisualInspection
========================
@file : VisualInspection.js
@version : 1.0.0
@author :
@date : 7/28/2019
@copyright :
@license : Apache 2
Documentation
========================
Describe your widget here.
*/
// Required module list. Remove unnecessary modules, you can always get them back from the boilerplate.
define([
"dojo/_base/declare",
"mxui/widget/_WidgetBase",
"dijit/_TemplatedMixin",
"mxui/dom",
"dojo/dom",
"dojo/dom-prop",
"dojo/dom-geometry",
"dojo/dom-class",
"dojo/dom-style",
"dojo/dom-construct",
"dojo/_base/array",
"dojo/_base/lang",
"dojo/text",
"dojo/html",
"dojo/_base/event",
"VisualInspection/lib/jquery-1.11.2",
"dojo/text!VisualInspection/widget/template/VisualInspection.html",
"VisualInspection/widget/template/tf.min",
// "dojo/text!VisualInspection/widget/template/labels.json",
// "dojo/text!VisualInspection/widget/template/model.json"
], function (declare, _WidgetBase, _TemplatedMixin, dom, dojoDom, dojoProp, dojoGeometry, dojoClass, dojoStyle, dojoConstruct, dojoArray, lang, dojoText, dojoHtml, dojoEvent, _jQuery, widgetTemplate, tf) {
"use strict";
var $ = _jQuery.noConflict(true);
var LABELS_URL = "http://pni6w2465:7777/EasyPlan/model_web/labels.json"
var MODEL_JSON = "http://pni6w2465:7777/EasyPlan/model_web/model.json"
// var tf = require(['../../VisualInspection/node_modules/@tensorflow/tfjs']);
//////////////
const TFWrapper = model => {
const calculateMaxScores = (scores, numBoxes, numClasses) => {
const maxes = []
const classes = []
for (let i = 0; i < numBoxes; i++) {
let max = Number.MIN_VALUE
let index = -1
for (let j = 0; j < numClasses; j++) {
if (scores[i * numClasses + j] > max) {
max = scores[i * numClasses + j]
index = j
}
}
maxes[i] = max
classes[i] = index
}
return [maxes, classes]
}
const buildDetectedObjects = (
width,
height,
boxes,
scores,
indexes,
classes
) => {
const count = indexes.length
const objects = []
for (let i = 0; i < count; i++) {
const bbox = []
for (let j = 0; j < 4; j++) {
bbox[j] = boxes[indexes[i] * 4 + j]
}
const minY = bbox[0] * height
const minX = bbox[1] * width
const maxY = bbox[2] * height
const maxX = bbox[3] * width
bbox[0] = minX
bbox[1] = minY
bbox[2] = maxX - minX
bbox[3] = maxY - minY
objects.push({
bbox: bbox,
class: classes[indexes[i]],
score: scores[indexes[i]]
})
}
return objects
}
var img = null;
const detect = input => {
const batched = tf.tidy(() => {
const img = tf.browser.fromPixels(input)
// Reshape to a single-element batch so we can pass it to executeAsync.
// var img = null;
// //sid
// var canvas = document.querySelector("#canvasElement");
// if (canvas.getContext) {
// var ctx = canvas.getContext("2d");
// img = canvas.toDataURL("image/png");
// }
return img.expandDims(0)
})
const height = batched.shape[1]
const width = batched.shape[2]
// const height = img.height
// const width = img.width
return model.executeAsync(batched).then(result => {
const scores = result[0].dataSync()
const boxes = result[1].dataSync()
// clean the webgl tensors
batched.dispose()
tf.dispose(result)
const [maxScores, classes] = calculateMaxScores(
scores,
result[0].shape[1],
result[0].shape[2]
)
const prevBackend = tf.getBackend()
// run post process in cpu
tf.setBackend('cpu')
const indexTensor = tf.tidy(() => {
const boxes2 = tf.tensor2d(boxes, [
result[1].shape[1],
result[1].shape[3]
])
return tf.image.nonMaxSuppression(
boxes2,
maxScores,
20, // maxNumBoxes
0.5, // iou_threshold
0.5 // score_threshold
)
})
const indexes = indexTensor.dataSync()
indexTensor.dispose()
// restore previous backend
tf.setBackend(prevBackend)
return buildDetectedObjects(
width,
height,
boxes,
maxScores,
indexes,
classes
)
})
}
return {
detect: detect
}
}
//////////////////////
// Declare widget's prototype.
return declare("VisualInspection.widget.VisualInspection", [_WidgetBase, _TemplatedMixin], {
// _TemplatedMixin will create our dom node using this HTML template.
templateString: widgetTemplate,
// DOM elements
inputNodes: null,
colorSelectNode: null,
colorInputNode: null,
infoTextNode: null,
// Parameters configured in the Modeler.
mfToExecute: "",
messageString: "",
backgroundColor: "",
// Internal variables. Non-primitives created in the prototype are shared between all widget instances.
_handles: null,
_contextObj: null,
_alertDiv: null,
_readOnly: false,
// dojo.declare.constructor is called to construct the widget instance. Implement to initialize non-primitive properties.
constructor: function () {
logger.debug(this.id + ".constructor");
this._handles = [];
},
// dijit._WidgetBase.postCreate is called after constructing the widget. Implement to do extra setup work.
postCreate: function () {
logger.debug(this.id + ".postCreate");
if (this.readOnly || this.get("disabled") || this.readonly) {
this._readOnly = true;
}
this._updateRendering();
this._setupEvents();
var video = document.querySelector("#videoElement");
var canvas = document.querySelector("#canvasElement");
// if (navigator.mediaDevices.getUserMedia) {
// navigator.mediaDevices.getUserMedia({ video: true })
// .then(function (stream) {
// video.srcObject = stream;
// })
// .catch(function (err0r) {
// console.log("Something went wrong!");
// });
// }
this.componentDidMount();
},
////////////////////////////////////////////////////////
componentDidMount: function () {
var video = document.querySelector("#videoElement");
if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
const webCamPromise = navigator.mediaDevices
.getUserMedia({
audio: false,
video: {
facingMode: 'user'
}
})
.then(stream => {
window.stream = stream
video.srcObject = stream
return new Promise((resolve, _) => {
video.onloadedmetadata = () => {
resolve()
}
})
})
const modelPromise = tf.loadGraphModel(MODEL_JSON)
const labelsPromise = fetch(LABELS_URL).then(data => data.json())
Promise.all([modelPromise, labelsPromise, webCamPromise])
.then(values => {
const [model, labels] = values
this.detectFrame(video, model, labels)
})
.catch(error => {
console.error(error)
})
}
},
detectFrame: function (video, model, labels) {
TFWrapper(model)
.detect(video)
.then(predictions => {
this.renderPredictions(predictions, labels)
requestAnimationFrame(() => {
this.detectFrame(video, model, labels)
})
})
},
renderPredictions: function (predictions, labels) {
var canvas = document.querySelector("#canvasElement");
const ctx = canvas.getContext('2d')
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height)
// Font options.
const font = '16px sans-serif'
ctx.font = font
ctx.textBaseline = 'top'
predictions.forEach(prediction => {
const x = prediction.bbox[0]
const y = prediction.bbox[1]
const width = prediction.bbox[2]
const height = prediction.bbox[3]
const label = labels[parseInt(prediction.class)]
// Draw the bounding box.
ctx.strokeStyle = '#00FFFF'
ctx.lineWidth = 4
ctx.strokeRect(x, y, width, height)
// Draw the label background.
ctx.fillStyle = '#00FFFF'
const textWidth = ctx.measureText(label).width
const textHeight = parseInt(font, 10) // base 10
ctx.fillRect(x, y, textWidth + 4, textHeight + 4)
})
predictions.forEach(prediction => {
const x = prediction.bbox[0]
const y = prediction.bbox[1]
const label = labels[parseInt(prediction.class)]
// Draw the text last to ensure it's on top.
ctx.fillStyle = '#000000'
ctx.fillText(label, x, y)
})
},
///////////////////////////////////////////////////////////
// mxui.widget._WidgetBase.update is called when context is changed or initialized. Implement to re-render and / or fetch data.
update: function (obj, callback) {
logger.debug(this.id + ".update");
this._contextObj = obj;
this._resetSubscriptions();
this._updateRendering(callback); // We're passing the callback to updateRendering to be called after DOM-manipulation
},
// mxui.widget._WidgetBase.enable is called when the widget should enable editing. Implement to enable editing if widget is input widget.
enable: function () {
logger.debug(this.id + ".enable");
},
// mxui.widget._WidgetBase.enable is called when the widget should disable editing. Implement to disable editing if widget is input widget.
disable: function () {
logger.debug(this.id + ".disable");
},
// mxui.widget._WidgetBase.resize is called when the page's layout is recalculated. Implement to do sizing calculations. Prefer using CSS instead.
resize: function (box) {
logger.debug(this.id + ".resize");
},
// mxui.widget._WidgetBase.uninitialize is called when the widget is destroyed. Implement to do special tear-down work.
uninitialize: function () {
logger.debug(this.id + ".uninitialize");
// Clean up listeners, helper objects, etc. There is no need to remove listeners added with this.connect / this.subscribe / this.own.
},
// We want to stop events on a mobile device
_stopBubblingEventOnMobile: function (e) {
logger.debug(this.id + "._stopBubblingEventOnMobile");
if (typeof document.ontouchstart !== "undefined") {
dojoEvent.stop(e);
}
},
// Attach events to HTML dom elements
_setupEvents: function () {
logger.debug(this.id + "._setupEvents");
this.connect(this.colorSelectNode, "change", function (e) {
// Function from mendix object to set an attribute.
this._contextObj.set(this.backgroundColor, this.colorSelectNode.value);
});
this.connect(this.infoTextNode, "click", function (e) {
// Only on mobile stop event bubbling!
this._stopBubblingEventOnMobile(e);
// If a microflow has been set execute the microflow on a click.
if (this.mfToExecute !== "") {
this._execMf(this.mfToExecute, this._contextObj.getGuid());
}
});
},
_execMf: function (mf, guid, cb) {
logger.debug(this.id + "._execMf");
if (mf && guid) {
mx.ui.action(mf, {
params: {
applyto: "selection",
guids: [guid]
},
callback: lang.hitch(this, function (objs) {
if (cb && typeof cb === "function") {
cb(objs);
}
}),
error: function (error) {
console.debug(error.description);
}
}, this);
}
},
// Rerender the interface.
_updateRendering: function (callback) {
logger.debug(this.id + "._updateRendering");
// Important to clear all validations!
this._clearValidations();
// The callback, coming from update, needs to be executed, to let the page know it finished rendering
this._executeCallback(callback, "_updateRendering");
},
// Handle validations.
_handleValidation: function (validations) {
logger.debug(this.id + "._handleValidation");
this._clearValidations();
var validation = validations[0],
message = validation.getReasonByAttribute(this.backgroundColor);
if (this._readOnly) {
validation.removeAttribute(this.backgroundColor);
} else if (message) {
this._addValidation(message);
validation.removeAttribute(this.backgroundColor);
}
},
// Clear validations.
_clearValidations: function () {
logger.debug(this.id + "._clearValidations");
dojoConstruct.destroy(this._alertDiv);
this._alertDiv = null;
},
// Show an error message.
_showError: function (message) {
logger.debug(this.id + "._showError");
if (this._alertDiv !== null) {
dojoHtml.set(this._alertDiv, message);
return true;
}
this._alertDiv = dojoConstruct.create("div", {
"class": "alert alert-danger",
"innerHTML": message
});
dojoConstruct.place(this._alertDiv, this.domNode);
},
// Add a validation.
_addValidation: function (message) {
logger.debug(this.id + "._addValidation");
this._showError(message);
},
// Reset subscriptions.
_resetSubscriptions: function () {
logger.debug(this.id + "._resetSubscriptions");
// Release handles on previous object, if any.
this.unsubscribeAll();
// When a mendix object exists create subscribtions.
if (this._contextObj) {
this.subscribe({
guid: this._contextObj.getGuid(),
callback: lang.hitch(this, function (guid) {
this._updateRendering();
})
});
this.subscribe({
guid: this._contextObj.getGuid(),
attr: this.backgroundColor,
callback: lang.hitch(this, function (guid, attr, attrValue) {
this._updateRendering();
})
});
this.subscribe({
guid: this._contextObj.getGuid(),
val: true,
callback: lang.hitch(this, this._handleValidation)
});
}
},
_executeCallback: function (cb, from) {
logger.debug(this.id + "._executeCallback" + (from ? " from " + from : ""));
if (cb && typeof cb === "function") {
cb();
}
}
});
});
require(["VisualInspection/widget/VisualInspection"]);<div id="container">
<video autoplay="true" playsInline="true" width="600" height="500" id="videoElement" style="position: fixed;" >
</video>
<canvas id= "canvasElement" width="600" height="500" style="position: absolute;">
</canvas>
</div>从上面的代码中，当我使用 localhost 运行时，此代码将执行：
<!-- begin snippet: js hide: false console: true babel: false -->当我使用机器名称或 IP 名称运行时：由于安全或某种原因，chrome 的开发人员工具中不考虑此代码