opencv EstimateAffinePartial2d：所需的输入格式是什么？

Question

我试图在“目标”图像中找到“模板”图像，并找到从模板图像坐标到目标图像坐标的变换。

我在this tutorial 之后取得了一些成功，并使用 ORB 查找关键点，然后使用蛮力匹配器查找相似的关键点。在相似的关键点中，该代码根据 Lowe 的比率测试进一步过滤了良好的匹配。剩余的“好”关键点与estimateAffinePartial2D 一起使用，以找到一组关键点之间的变换。

我在 python 中有一个工作版本：

import sys
import numpy as np
import cv2.cv2 as cv2

# with the name image.jpg
img1 = cv2.imread('score_overlay_2021_1280.png')
img2 = cv2.imread('2021/frame-00570.jpg')

orb = cv2.ORB_create(nfeatures=1000) # Increasing nfeatures to get more keypoints

kp1, des1 = orb.detectAndCompute(img1, None)
kp2, des2 = orb.detectAndCompute(img2, None)


matcher = cv2.BFMatcher()
matches = matcher.knnMatch(des1, des2, k=2)

# Apply ratio test
good = []
for m, n in matches:
    if m.distance < 0.75 * n.distance:
        good.append(m)

print("found {} matches".format(len(good)))

if len(good) < 7:
    print("Not enough good keypoint matches between template and image")
    sys.exit(1)

src_pts = np.float32([kp1[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
dst_pts = np.float32([kp2[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
t = cv2.estimateAffinePartial2D(src_pts, dst_pts)

print(t)

我正在尝试使用 gocv 绑定为 opencv 在 go(lang) 中编写与上面相同的代码，但是我必须错误地将某些内容传递给estimateAffinePartial2D

这是错误：

libc++abi.dylib: terminating with uncaught exception of type cv::Exception: OpenCV(4.5.3) /tmp/opencv-20210728-84579-13worgs/opencv-4.5.3/modules/calib3d/src/ptsetreg.cpp:1108: error: (-215:Assertion failed) count >= 0 && to.checkVector(2) == count in function 'estimateAffinePartial2D'

这是产生上述错误的几乎相同的 go 代码

package main

import (
    "fmt"
    "log"
    "os"

    "gocv.io/x/gocv"
)

func run() error {
    var nfeatures = 1000                    // default is 500
    var scaleFactor float32 = 1.2           // default 1.2
    var nlevels = 8                         // default 8
    var edgeThreshold = 31                  // default 32
    var firstLevel = 0                      // default 0
    var WtaK = 2                            // default 2
    var scoreType = gocv.ORBScoreTypeHarris // default ORBScoreTypeHarris
    var patchSize = 31                      // default 31
    var fastThreshold = 20                  // default 20

    algo := gocv.NewORBWithParams(nfeatures, scaleFactor, nlevels, edgeThreshold, firstLevel, WtaK, scoreType, patchSize, fastThreshold)

    overlayFile := "score_overlay_2021_1280.png"
    overlay := gocv.IMRead(overlayFile, gocv.IMReadColor)
    if overlay.Empty() {
        return fmt.Errorf("unable to load %s", overlayFile)
    }
    kp1, des1 := algo.DetectAndCompute(overlay, gocv.NewMat())

    matcher := gocv.NewBFMatcher()

    frameFile := "frame-00570.jpg"
    img := gocv.IMRead(frameFile, gocv.IMReadColor)
    if img.Empty() {
        return fmt.Errorf("unable to load %s", frameFile)
    }
    kp2, des2 := algo.DetectAndCompute(img, gocv.NewMat())

    matches := matcher.KnnMatch(des1, des2, 4)

    // Store all the good matches as per Lowe's ratio test
    goodMatches := make([]gocv.DMatch, 0)
    for _, submatches := range matches {
        if submatches[0].Distance < 0.7*submatches[1].Distance {
            goodMatches = append(goodMatches, submatches[0])
        }
    }
    const MinGoodMatches = 6
    if len(goodMatches) < MinGoodMatches {
        return fmt.Errorf("only found %d matches, need at least %d", len(goodMatches), MinGoodMatches)
    }

    kp1vec := kp2Point2f(kp1)
    kp2vec := kp2Point2f(kp2)

    t := gocv.EstimateAffinePartial2D(kp1vec, kp2vec)

    log.Printf("T: %#v", t)
    return nil
}

func kp2Point2f(kp []gocv.KeyPoint) gocv.Point2fVector {
    kp2f := make([]gocv.Point2f, 0)
    for _, kp := range kp {
        kp2f = append(kp2f, gocv.Point2f{
            X: float32(kp.X),
            Y: float32(kp.Y),
        })
    }
    return gocv.NewPoint2fVectorFromPoints(kp2f)
}

func main() {
    if err := run(); err != nil {
        log.Printf("Error: %s", err)
        os.Exit(1)
    }
}

模板图片： https://github.com/TechplexEngineer/frc-livescore/blob/orb-test/orbtest/score_overlay_2021_1280.png

框架： https://github.com/TechplexEngineer/frc-livescore/blob/orb-test/orbtest/2021/frame-00570.jpg

Answer 1

原来上面的 go(lang) 代码没有正确应用 Lowe 比率测试。

这是工作代码：

package main

import (
    "fmt"
    "image"
    "log"
    "os"

    "gocv.io/x/gocv"
)

func run() error {
    var nfeatures = 1000                    // default is 500
    var scaleFactor float32 = 1.2           // default 1.2
    var nlevels = 8                         // default 8
    var edgeThreshold = 31                  // default 32
    var firstLevel = 0                      // default 0
    var WtaK = 2                            // default 2
    var scoreType = gocv.ORBScoreTypeHarris // default ORBScoreTypeHarris
    var patchSize = 31                      // default 31
    var fastThreshold = 20                  // default 20

    algo := gocv.NewORBWithParams(nfeatures, scaleFactor, nlevels, edgeThreshold, firstLevel, WtaK, scoreType, patchSize, fastThreshold)

    overlayFile := "score_overlay_2021_1280.png"
    overlay := gocv.IMRead(overlayFile, gocv.IMReadColor)
    if overlay.Empty() {
        return fmt.Errorf("unable to load %s", overlayFile)
    }
    kp1, des1 := algo.DetectAndCompute(overlay, gocv.NewMat())

    matcher := gocv.NewBFMatcher()

    frameFile := "frame-00570.jpg"
    img := gocv.IMRead(frameFile, gocv.IMReadColor)
    if img.Empty() {
        return fmt.Errorf("unable to load %s", frameFile)
    }

    newWidth := 1280.0
    newHeight := newWidth*(float64(img.Rows())/float64(img.Cols()))
    gocv.Resize(img, &img, image.Pt(int(newWidth), int(newHeight)), 0,0, gocv.InterpolationDefault)

    kp2, des2 := algo.DetectAndCompute(img, gocv.NewMat())

    matches := matcher.KnnMatch(des1, des2, 4)

    // Store all the good matches as per Lowe's ratio test
    goodMatches := make([]gocv.DMatch, 0)
    for _, submatches := range matches {
        if submatches[0].Distance < 0.75 * submatches[1].Distance {
            goodMatches = append(goodMatches, submatches[0])
        }
    }
    const MinGoodMatches = 7
    if len(goodMatches) < MinGoodMatches {
        return fmt.Errorf("only found %d matches, need at least %d", len(goodMatches), MinGoodMatches)
    }

    kp1f := make([]gocv.Point2f, 0)
    for _, gm := range goodMatches {
        kp1f = append(kp1f, gocv.Point2f{
            X: float32(kp1[gm.QueryIdx].X),
            Y: float32(kp1[gm.QueryIdx].Y),
        })
    }
    kp1vec := gocv.NewPoint2fVectorFromPoints(kp1f)

    kp2f := make([]gocv.Point2f, 0)
    for _, gm := range goodMatches {
        kp2f = append(kp2f, gocv.Point2f{
            X: float32(kp2[gm.TrainIdx].X),
            Y: float32(kp2[gm.TrainIdx].Y),
        })
    }
    kp2vec := gocv.NewPoint2fVectorFromPoints(kp2f)

    log.Printf("kp1:%d kp2:%d", kp1vec.Size(),kp2vec.Size())

    t := gocv.EstimateAffinePartial2D(kp1vec, kp2vec)
    
    scale := t.GetDoubleAt(0,0)
    tx := t.GetDoubleAt(0,2)
    ty := t.GetDoubleAt(1,2)

    log.Printf("Scale: %f Tx: %f Ty: %f", scale, tx, ty)
    return nil
}

func main() {
    if err := run(); err != nil {
        log.Printf("Error: %s", err)
        os.Exit(1)
    }
}