p5.j​​s：如何使用 face-api.js 中的坐标进行绘制？

Question

我正在尝试检索 face-api.js（实时网络摄像头）中的 68 个面部标志，并使用这些点在 p5.js 中绘制 curveVertex()。到目前为止，我可以检索地标，但在 draw() 中使用它时遇到问题。

我正在使用https://github.com/WebDevSimplified/Face-Detection-JavaScript 的此代码进行人脸检测，并尝试在其中添加一些 p5.js 代码：

const video = document.getElementById('video')

Promise.all([
  faceapi.nets.tinyFaceDetector.loadFromUri('/models'),
  faceapi.nets.faceLandmark68Net.loadFromUri('/models'),
  faceapi.nets.faceRecognitionNet.loadFromUri('/models'),
  faceapi.nets.faceExpressionNet.loadFromUri('/models')
]).then(startVideo)

function startVideo() {
  navigator.getUserMedia({
      video: {}
    },
    stream => video.srcObject = stream,
    err => console.error(err)
  )

}

video.addEventListener('playing', () => {
  const canvas = faceapi.createCanvasFromMedia(video)
  document.body.append(canvas)
  const displaySize = {
    width: video.width,
    height: video.height
  }
  faceapi.matchDimensions(canvas, displaySize)

  setInterval(async () => {
    const detections = await faceapi.detectAllFaces(video, new faceapi.TinyFaceDetectorOptions()).withFaceLandmarks()
    const resizedDetections = faceapi.resizeResults(detections, displaySize)
    canvas.getContext('2d').clearRect(0, 0, canvas.width, canvas.height)
    faceapi.draw.drawFaceLandmarks(canvas, resizedDetections)
    landmarks = await faceapi.detectFaceLandmarks(video)
    landmarkPositions = landmarks.positions
    // console.log(landmarkPositions[0]);

  }, 100)

})

//p5.js
function setup() {
  var myCanvas = createCanvas(windowWidth, windowHeight);
      myCanvas.parent("overlay");
  angleMode(DEGREES);
}

function draw() {
  // background(0);
  stroke(255);
  noFill();
  strokeWeight(4);

  beginShape();
  //test
  curveVertex(100, 200);
  curveVertex(150, 50);
  curveVertex(250, 60);
  curveVertex(300, 200);
  curveVertex(300, 200);
  //get landmark positions
  curveVertex(landmarkPositions[0]);
  // curveVertex(landmarkPositions[1]),
  // curveVertex(landmarkPositions[2]),
  // curveVertex(landmarkPositions[3])

  endShape(CLOSE);
}

body {
      margin: 0;
      padding: 0;
      width: 100vw;
      height: 100vh;
    }

    canvas {
      position: absolute;
    }

    #overlay{
      position: absolute;
      z-index: 1;
    }

<div id="overlay"></div>
<video id="video" width="720" height="560" autoplay muted></video>

如何在curveVertex()中实时使用landmarkPositions？

非常感谢任何帮助，在此先感谢！

Answer 1

beginShape()/curveVertex()/endShape() 你走在正确的轨道上，它应该（理论上）只是遍历每个地标位置并传递 x,y 坐标进行绘制的问题。

目前您似乎直接通过地标：curveVertex(landmarkPositions[0]); 而curveVertex() 需要 x、y 坐标。

类似的东西应该可以工作：

curveVertex(landmarkPositions[0].x, landmarkPositions[0].y); 因为根据face-api documentation 一个Point 具有x,y 属性

您可以使用 for 循环遍历每个点：

beginShape();
for(let i = 0 ; i < landmarkPositions.length; i++){
  //get landmark position
  let position = landmarkPositions[i];
  curveVertex(position.x, position.y);
}
endShape(CLOSE);

另外请注意，FaceLandmark68 在positions 数组之上还带有一些便利函数，例如：

• getJawOutline()
• getLeftEye()
• getLeftEyeBrow()
• getMouth()
• getNose()
• getRightEye()
• getRightEyeBrow()

我猜你想尝试曲线而不是直线？ 如果您只是想渲染地标线face-api can do that too。这是他们的示例 sn-p：

const detectionsWithLandmarks = await faceapi
  .detectAllFaces(input)
  .withFaceLandmarks()

// resize the detected boxes and landmarks in case your displayed image has a different size then the original
const detectionsWithLandmarksForSize = faceapi.resizeResults(detectionsWithLandmarks, { width: input.width, height: input.height })
// draw them into a canvas
const canvas = document.getElementById('overlay')
canvas.width = input.width
canvas.height = input.height
faceapi.drawLandmarks(canvas, detectionsWithLandmarks, { drawLines: true })

这一行尤其是您所追求的：faceapi.drawLandmarks(canvas, detectionsWithLandmarks, { drawLines: true })

如果您已经将 p5.js 画布存储为 myCanvas，只需将 myCanvas.elt 传递给 faceapi 即可获得 p5.js HTML <canvas/> 元素

更新：

这是一个带有 cmets 中提到的注释的 sn-p：基本上使用 p5.js 和 faceapi

不幸的是，我无法轻松地将模型和 faceapi.js 上传到 stackoverflow sn-ps 或在线 p5.js 编辑器，但是您应该能够在您的 Face-Detection-JavaScript 副本中使用此脚本：

// p5.js canvas and it's HTML <canvas/> element
let p5Canvas;
let p5CanvasElement;
// p5.js capture and it's HTML <video/> element
let capture;
let captureElement;
// store dimenions formatted for faceapi
let displaySize;

function setup(){
  // create canvas (same size as default camera)..and to it's HTML element (.elt)
  p5Canvas = createCanvas(640, 480);
  p5CanvasElement = p5Canvas.elt;
  // create capture and reference it's HTML element (.elt)
  capture = createCapture(VIDEO);
  captureElement = capture.elt;
  // hide video
  capture.hide();
  // setup faceapi dimensions
  displaySize = { width: width, height: height };
  faceapi.matchDimensions(p5Canvas, displaySize);

  // optional: match setInterval(..., 100) -> 10fps from the example
  frameRate(10);
  noFill();
  // trigger model loading
  loadModels();
  // pause P5's update loop until the video is ready ('play' event)
  text("loading models", width * 0.5, height * 0.5);
  noLoop();
  
}

function loadModels(){
  Promise.all([
    faceapi.nets.tinyFaceDetector.loadFromUri('/models'),
    faceapi.nets.faceLandmark68Net.loadFromUri('/models'),
    faceapi.nets.faceRecognitionNet.loadFromUri('/models'),
    faceapi.nets.faceExpressionNet.loadFromUri('/models')
  ])
  // noLoop() was called in setup, pausing draw() while we load, we resume here once models are loaded
  .then(loop);
}
// make draw async to await faceapi results
async function draw(){
  const detections = await faceapi.detectAllFaces(captureElement, new faceapi.TinyFaceDetectorOptions()).withFaceLandmarks().withFaceExpressions()
  const resizedDetections = faceapi.resizeResults(detections, displaySize)
  // render capture
  image(capture, 0, 0);
  
  // optional if mouse pressed debug draw face api in p5.js canvas
  if(mouseIsPressed){
    faceapi.draw.drawDetections(p5CanvasElement, resizedDetections);
    faceapi.draw.drawFaceLandmarks(p5CanvasElement, resizedDetections);
    faceapi.draw.drawFaceExpressions(p5CanvasElement, resizedDetections);
  }
  // default draw curveVertex()
  else{
    // skip this frame if there are no detections
    if(resizedDetections.length == 0) return;
    
    // example getting all landmarks (connected)
    // const landmarkPositions = resizedDetections[0].landmarks.positions;

    // example getting a subset (e.g. getMouth(), getNose(), etc.)
    const landmarkPositions = getMouth(resizedDetections[0].landmarks.positions);
    
    stroke(255);
    beginShape();
    for(let i = 0 ; i < landmarkPositions.length; i++){
      const position = landmarkPositions[i];
      curveVertex(position.x, position.y);
    }
    endShape(CLOSE);
  }
  
}

function getJawOutline(positions) {
  return positions.slice(0, 17);
}

function getLeftEyeBrow(positions) {
  return positions.slice(17, 22);
}

function getRightEyeBrow(positions) {
  return positions.slice(22, 27);
}

function getNose(positions) {
  return positions.slice(27, 36);
}

function getLeftEye(positions) {
  return positions.slice(36, 42);
}

function getRightEye(positions) {
  return positions.slice(42, 48);
}

function getMouth(positions) {
  return positions.slice(48, 68);
}

代码已注释，您可以选择为脸部的一部分（例如 getMouth()、getNose() 等）或整个脸部（尽管连接脸部元素）绘制位置，尽管您可能会这样做想探索（例如Matty Mariansky's experiments）