如何将图像的单个字母旋转到正确的方向以获得最佳 OCR？

Question

在my previous question，我改造了这张图片：

进入这个：

Tesseract OCR 解释为：

1O351

在图像周围放置一个框架

实际上改善了 OCR 结果。

 1CB51

但是，我需要所有 5 个字符来正确进行 OCR，因此作为一个实验，我使用 Paint.NET 将每个单独的字母旋转并对齐到正确的方向：

得出正确答案：

1CB52

我将如何在 C# 中执行此更正？

我对各种文本对齐算法进行了一些研究，但它们都假设源图像中存在 文本行，您可以从中得出旋转角度，但是其中已经包含了字母之间适当的间距和方向关系。

Answer 1

您可以使用以下code project article 中的代码来分割每个单独的字符。但是，当尝试单独校正这些字符时，您得到的任何结果都不会很好，因为没有太多信息可供使用。

我尝试使用AForge.NETs HoughLineTransformation class，得到的角度范围在 80 - 90 度之间。所以我尝试使用以下代码来校正它们：

private static Bitmap DeskewImageByIndividualChars(Bitmap targetBitmap)
{
    IDictionary<Rectangle, Bitmap> characters = new CCL().Process(targetBitmap);

    using (Graphics g = Graphics.FromImage(targetBitmap))
    {
        foreach (var character in characters)
        {
            double angle;

            BitmapData bitmapData = character.Value.LockBits(new Rectangle(Point.Empty, character.Value.Size), ImageLockMode.ReadWrite, PixelFormat.Format8bppIndexed);
            try
            {
                HoughLineTransformation hlt = new HoughLineTransformation();
                hlt.ProcessImage(bitmapData);

                angle = hlt.GetLinesByRelativeIntensity(0.5).Average(l => l.Theta);
            }
            finally
            {
                character.Value.UnlockBits(bitmapData);
            }

            using (Bitmap bitmap = RotateImage(character.Value, 90 - angle, Color.White))
            {
                g.DrawImage(bitmap, character.Key.Location);
            }
        }
    }

    return targetBitmap;
}

使用RotateImage method taken from here. 但是，结果似乎并不是最好的。也许你可以尝试让它们变得更好。

这是代码项目文章中的代码供您参考。我对其进行了一些更改，使其表现得更安全一些，例如在LockBits 周围添加try-finally 并使用using 语句正确处理对象等。

using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.Linq;

namespace ConnectedComponentLabeling
{
    public class CCL
    {
        private Bitmap _input;
        private int[,] _board;

        public IDictionary<Rectangle, Bitmap> Process(Bitmap input)
        {
            _input = input;
            _board = new int[_input.Width, _input.Height];

            Dictionary<int, List<Pixel>> patterns = Find();
            var images = new Dictionary<Rectangle, Bitmap>();

            foreach (KeyValuePair<int, List<Pixel>> pattern in patterns)
            {
                using (Bitmap bmp = CreateBitmap(pattern.Value))
                {
                    images.Add(GetBounds(pattern.Value), (Bitmap)bmp.Clone());
                }
            }

            return images;
        }

        protected virtual bool CheckIsBackGround(Pixel currentPixel)
        {
            return currentPixel.color.A == 255 && currentPixel.color.R == 255 && currentPixel.color.G == 255 && currentPixel.color.B == 255;
        }

        private unsafe Dictionary<int, List<Pixel>> Find()
        {
            int labelCount = 1;
            var allLabels = new Dictionary<int, Label>();

            BitmapData imageData = _input.LockBits(new Rectangle(0, 0, _input.Width, _input.Height), ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);
            try
            {
                int bytesPerPixel = 3;

                byte* scan0 = (byte*)imageData.Scan0.ToPointer();
                int stride = imageData.Stride;

                for (int i = 0; i < _input.Height; i++)
                {
                    byte* row = scan0 + (i * stride);

                    for (int j = 0; j < _input.Width; j++)
                    {
                        int bIndex = j * bytesPerPixel;
                        int gIndex = bIndex + 1;
                        int rIndex = bIndex + 2;

                        byte pixelR = row[rIndex];
                        byte pixelG = row[gIndex];
                        byte pixelB = row[bIndex];

                        Pixel currentPixel = new Pixel(new Point(j, i), Color.FromArgb(pixelR, pixelG, pixelB));

                        if (CheckIsBackGround(currentPixel))
                        {
                            continue;
                        }

                        IEnumerable<int> neighboringLabels = GetNeighboringLabels(currentPixel);
                        int currentLabel;

                        if (!neighboringLabels.Any())
                        {
                            currentLabel = labelCount;
                            allLabels.Add(currentLabel, new Label(currentLabel));
                            labelCount++;
                        }
                        else
                        {
                            currentLabel = neighboringLabels.Min(n => allLabels[n].GetRoot().Name);
                            Label root = allLabels[currentLabel].GetRoot();

                            foreach (var neighbor in neighboringLabels)
                            {
                                if (root.Name != allLabels[neighbor].GetRoot().Name)
                                {
                                    allLabels[neighbor].Join(allLabels[currentLabel]);
                                }
                            }
                        }

                        _board[j, i] = currentLabel;
                    }
                }
            }
            finally
            {
                _input.UnlockBits(imageData);
            }

            Dictionary<int, List<Pixel>> patterns = AggregatePatterns(allLabels);

            patterns = RemoveIntrusions(patterns, _input.Width, _input.Height);

            return patterns;
        }

        private Dictionary<int, List<Pixel>> RemoveIntrusions(Dictionary<int, List<Pixel>> patterns, int width, int height)
        {
            var patternsCleaned = new Dictionary<int, List<Pixel>>();

            foreach (var pattern in patterns)
            {
                bool bad = false;
                foreach (Pixel item in pattern.Value)
                {
                    //Horiz
                    if (item.Position.X == 0)
                        bad = true;

                    else if (item.Position.Y == width - 1)
                        bad = true;

                    //Vert
                    else if (item.Position.Y == 0)
                        bad = true;

                    else if (item.Position.Y == height - 1)
                        bad = true;
                }

                if (!bad)
                    patternsCleaned.Add(pattern.Key, pattern.Value);

            }

            return patternsCleaned;
        }

        private IEnumerable<int> GetNeighboringLabels(Pixel pix)
        {
            var neighboringLabels = new List<int>();

            for (int i = pix.Position.Y - 1; i <= pix.Position.Y + 2 && i < _input.Height - 1; i++)
            {
                for (int j = pix.Position.X - 1; j <= pix.Position.X + 2 && j < _input.Width - 1; j++)
                {
                    if (i > -1 && j > -1 && _board[j, i] != 0)
                    {
                        neighboringLabels.Add(_board[j, i]);
                    }
                }
            }

            return neighboringLabels;
        }

        private Dictionary<int, List<Pixel>> AggregatePatterns(Dictionary<int, Label> allLabels)
        {
            var patterns = new Dictionary<int, List<Pixel>>();

            for (int i = 0; i < _input.Height; i++)
            {
                for (int j = 0; j < _input.Width; j++)
                {
                    int patternNumber = _board[j, i];
                    if (patternNumber != 0)
                    {
                        patternNumber = allLabels[patternNumber].GetRoot().Name;

                        if (!patterns.ContainsKey(patternNumber))
                        {
                            patterns[patternNumber] = new List<Pixel>();
                        }

                        patterns[patternNumber].Add(new Pixel(new Point(j, i), Color.Black));
                    }
                }
            }

            return patterns;
        }

        private unsafe Bitmap CreateBitmap(List<Pixel> pattern)
        {
            int minX = pattern.Min(p => p.Position.X);
            int maxX = pattern.Max(p => p.Position.X);

            int minY = pattern.Min(p => p.Position.Y);
            int maxY = pattern.Max(p => p.Position.Y);

            int width = maxX + 1 - minX;
            int height = maxY + 1 - minY;

            Bitmap bmp = DrawFilledRectangle(width, height);

            BitmapData imageData = bmp.LockBits(new Rectangle(0, 0, bmp.Width, bmp.Height), ImageLockMode.ReadWrite, PixelFormat.Format24bppRgb);
            try
            {
                byte* scan0 = (byte*)imageData.Scan0.ToPointer();
                int stride = imageData.Stride;

                foreach (Pixel pix in pattern)
                {
                    scan0[((pix.Position.X - minX) * 3) + (pix.Position.Y - minY) * stride] = pix.color.B;
                    scan0[((pix.Position.X - minX) * 3) + (pix.Position.Y - minY) * stride + 1] = pix.color.G;
                    scan0[((pix.Position.X - minX) * 3) + (pix.Position.Y - minY) * stride + 2] = pix.color.R;
                }
            }
            finally
            {
                bmp.UnlockBits(imageData);
            }

            return bmp;
        }

        private Bitmap DrawFilledRectangle(int x, int y)
        {
            Bitmap bmp = new Bitmap(x, y);
            using (Graphics graph = Graphics.FromImage(bmp))
            {
                Rectangle ImageSize = new Rectangle(0, 0, x, y);
                graph.FillRectangle(Brushes.White, ImageSize);
            }

            return bmp;
        }

        private Rectangle GetBounds(List<Pixel> pattern)
        {
            var points = pattern.Select(x => x.Position);

            var x_query = points.Select(p => p.X);
            int xmin = x_query.Min();
            int xmax = x_query.Max();

            var y_query = points.Select(p => p.Y);
            int ymin = y_query.Min();
            int ymax = y_query.Max();

            return new Rectangle(xmin, ymin, xmax - xmin, ymax - ymin);
        }
    }
}

通过上面的代码，我得到了以下输入/输出：

如您所见，B 旋转得很好，但其他的则没有那么好。

---

尝试去歪斜单个字符的另一种方法是使用上面的分割例程找到那里的位置。然后将每个单独的字符分别传递给您的识别引擎，看看这是否会改善您的结果。

---

我使用以下方法从CCL 类内部使用List<Pixel> 查找字符的角度。它通过找到“左下”和“右下”点之间的角度来工作。我还没有测试过，如果角色反向旋转，它是否有效。

private double GetAngle(List<Pixel> pattern)
{
    var pixels = pattern.Select(p => p.Position).ToArray();

    Point bottomLeft = pixels.OrderByDescending(p => p.Y).ThenBy(p => p.X).First();
    Point rightBottom = pixels.OrderByDescending(p => p.X).ThenByDescending(p => p.Y).First();

    int xDiff = rightBottom.X - bottomLeft.X;
    int yDiff = rightBottom.Y - bottomLeft.Y;

    double angle = Math.Atan2(yDiff, xDiff) * 180 / Math.PI;

    return -angle;
}

请注意，我的绘图代码有点损坏，这就是为什么 5 在右侧被截断，但此代码产生以下输出：

请注意，B 和 5 的旋转幅度超出了您的预期，因为它们的曲率。

---

使用以下代码，通过从左右边缘获取角度，然后选择最佳的角度，旋转似乎更好。请注意，我只用需要顺时针旋转的字母对其进行了测试，所以如果它们需要逆时针旋转，它可能效果不佳。

这也会对像素进行“象限”，以便从其自己的象限中选择每个像素，以免获得两个太近的像素。

选择最佳角度的想法是，如果它们相似，目前彼此相差 1.5 度以内但可以轻松更新，则将它们平均。否则我们选择最接近零的那个。

private double GetAngle(List<Pixel> pattern, Rectangle bounds)
{
    int halfWidth = bounds.X + (bounds.Width / 2);
    int halfHeight = bounds.Y + (bounds.Height / 2);

    double leftEdgeAngle = GetAngleLeftEdge(pattern, halfWidth, halfHeight);
    double rightEdgeAngle = GetAngleRightEdge(pattern, halfWidth, halfHeight);

    if (Math.Abs(leftEdgeAngle - rightEdgeAngle) <= 1.5)
    {
        return (leftEdgeAngle + rightEdgeAngle) / 2d;
    }

    if (Math.Abs(leftEdgeAngle) > Math.Abs(rightEdgeAngle))
    {
        return rightEdgeAngle;
    }
    else
    {
        return leftEdgeAngle;
    }
}

private double GetAngleLeftEdge(List<Pixel> pattern, double halfWidth, double halfHeight)
{
    var topLeftPixels = pattern.Select(p => p.Position).Where(p => p.Y < halfHeight && p.X < halfWidth).ToArray();
    var bottomLeftPixels = pattern.Select(p => p.Position).Where(p => p.Y > halfHeight && p.X < halfWidth).ToArray();

    Point topLeft = topLeftPixels.OrderBy(p => p.X).ThenBy(p => p.Y).First();
    Point bottomLeft = bottomLeftPixels.OrderByDescending(p => p.Y).ThenBy(p => p.X).First();

    int xDiff = bottomLeft.X - topLeft.X;
    int yDiff = bottomLeft.Y - topLeft.Y;

    double angle = Math.Atan2(yDiff, xDiff) * 180 / Math.PI;

    return 90 - angle;
}

private double GetAngleRightEdge(List<Pixel> pattern, double halfWidth, double halfHeight)
{
    var topRightPixels = pattern.Select(p => p.Position).Where(p => p.Y < halfHeight && p.X > halfWidth).ToArray();
    var bottomRightPixels = pattern.Select(p => p.Position).Where(p => p.Y > halfHeight && p.X > halfWidth).ToArray();

    Point topRight = topRightPixels.OrderBy(p => p.Y).ThenByDescending(p => p.X).First();
    Point bottomRight = bottomRightPixels.OrderByDescending(p => p.X).ThenByDescending(p => p.Y).First();

    int xDiff = bottomRight.X - topRight.X;
    int yDiff = bottomRight.Y - topRight.Y;

    double angle = Math.Atan2(xDiff, yDiff) * 180 / Math.PI;

    return Math.Abs(angle);
}

这现在产生以下输出，我的绘图代码再次被破坏。请注意，C 看起来并没有很好地去歪斜，但仔细观察它只是它的形状导致了这种情况的发生。

---

我改进了绘图代码并尝试将字符放在同一基线上：

private static Bitmap DeskewImageByIndividualChars(Bitmap bitmap)
{
    IDictionary<Rectangle, Tuple<Bitmap, double>> characters = new CCL().Process(bitmap);

    Bitmap deskewedBitmap = new Bitmap(bitmap.Width, bitmap.Height, bitmap.PixelFormat);
    deskewedBitmap.SetResolution(bitmap.HorizontalResolution, bitmap.VerticalResolution);

    using (Graphics g = Graphics.FromImage(deskewedBitmap))
    {
        g.FillRectangle(Brushes.White, new Rectangle(Point.Empty, deskewedBitmap.Size));

        int baseLine = characters.Max(c => c.Key.Bottom);
        foreach (var character in characters)
        {
            int y = character.Key.Y;
            if (character.Key.Bottom != baseLine)
            {
                y += (baseLine - character.Key.Bottom - 1);
            }

            using (Bitmap characterBitmap = RotateImage(character.Value.Item1, character.Value.Item2, Color.White))
            {
                g.DrawImage(characterBitmap, new Point(character.Key.X, y));
            }
        }
    }

    return deskewedBitmap;
}

这将产生以下输出。请注意，由于要使用预旋转底部来解决每个字符，因此它们不在完全相同的基线上。需要使用旋转后的基线来改进代码。在做基线之前对图像进行阈值处理也会有所帮助。

另一个改进是计算每个旋转字符位置的Right，因此在绘制下一个字符时它不会与前一个重叠并切断位。因为正如您在输出中看到的那样，2 略微切入了5。

现在的输出与 OP 中手动创建的非常相似。