Handle OCR / Computer Vision result to match a receipt structure

C

2

8

I'm using Microsoft Computer Vision to read receipts.

The result I get is ordered into regions that are grouped by columns, e.g. quantities, product names, amount are in three different regions.

I would to prefer if the whole list of products is one region and that each line is a product.

Is there any way to configure the Computer Vision to accomplish this, or maybe more likely are there any good techniques or libraries that one can use in a post-processing of the result since the positions of all the words are available.

Bellow is the image of the receipt and the result from the computer vision.

{
  "language": "sv",
  "textAngle": 2.0999999999999632,
  "orientation": "Up",
  "regions": [
    {
      "boundingBox": "1012,450,660,326",
      "lines": [
        {
          "boundingBox": "1362,450,76,30",
          "words": [
            {
              "boundingBox": "1362,450,76,30",
              "text": "JULA"
            }
          ]
        },
        {
          "boundingBox": "1207,486,465,49",
          "words": [
            {
              "boundingBox": "1207,502,172,33",
              "text": "Ekslinsan"
            },
            {
              "boundingBox": "1400,497,51,30",
              "text": "3B,"
            },
            {
              "boundingBox": "1479,491,95,33",
              "text": "25467"
            },
            {
              "boundingBox": "1595,486,77,32",
              "text": "VALA"
            }
          ]
        },
        {
          "boundingBox": "1304,539,265,38",
          "words": [
            {
              "boundingBox": "1304,539,265,38",
              "text": "SE5S6944785601"
            }
          ]
        },
        {
          "boundingBox": "1245,584,369,44",
          "words": [
            {
              "boundingBox": "1245,594,148,34",
              "text": "Telefon:"
            },
            {
              "boundingBox": "1421,584,193,37",
              "text": "042-324040"
            }
          ]
        },
        {
          "boundingBox": "1012,695,269,35",
          "words": [
            {
              "boundingBox": "1012,702,75,28",
              "text": "Kund"
            },
            {
              "boundingBox": "1109,695,172,33",
              "text": "072202787"
            }
          ]
        },
        {
          "boundingBox": "1109,738,289,38",
          "words": [
            {
              "boundingBox": "1109,744,133,32",
              "text": "LILLVIK"
            },
            {
              "boundingBox": "1265,738,133,32",
              "text": "ANDREAS"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1085,845,14,516",
      "lines": [
        {
          "boundingBox": "1090,845,9,29",
          "words": [
            {
              "boundingBox": "1090,845,9,29",
              "text": "1"
            }
          ]
        },
        {
          "boundingBox": "1087,1037,9,28",
          "words": [
            {
              "boundingBox": "1087,1037,9,28",
              "text": "1"
            }
          ]
        },
        {
          "boundingBox": "1086,1133,9,27",
          "words": [
            {
              "boundingBox": "1086,1133,9,27",
              "text": "I"
            }
          ]
        },
        {
          "boundingBox": "1085,1332,9,29",
          "words": [
            {
              "boundingBox": "1085,1332,9,29",
              "text": "1"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1122,839,454,573",
      "lines": [
        {
          "boundingBox": "1128,839,173,33",
          "words": [
            {
              "boundingBox": "1128,843,36,29",
              "text": "ST"
            },
            {
              "boundingBox": "1186,839,115,30",
              "text": "661107"
            }
          ]
        },
        {
          "boundingBox": "1127,879,389,41",
          "words": [
            {
              "boundingBox": "1127,887,232,33",
              "text": "VERKTYGSLÅDR"
            },
            {
              "boundingBox": "1382,883,36,28",
              "text": "JC"
            },
            {
              "boundingBox": "1441,882,16,26",
              "text": "5"
            },
            {
              "boundingBox": "1481,879,35,28",
              "text": "ÅR"
            }
          ]
        },
        {
          "boundingBox": "1126,935,173,34",
          "words": [
            {
              "boundingBox": "1126,940,36,29",
              "text": "ST"
            },
            {
              "boundingBox": "1187,935,112,32",
              "text": "181460"
            }
          ]
        },
        {
          "boundingBox": "1126,967,450,50",
          "words": [
            {
              "boundingBox": "1126,987,75,30",
              "text": "BORR"
            },
            {
              "boundingBox": "1224,977,193,35",
              "text": "GLAS/KRKEL"
            },
            {
              "boundingBox": "1440,974,16,27",
              "text": "ø"
            },
            {
              "boundingBox": "1482,971,34,27",
              "text": "10"
            },
            {
              "boundingBox": "1539,967,37,28",
              "text": "MM"
            }
          ]
        },
        {
          "boundingBox": "1125,1027,173,37",
          "words": [
            {
              "boundingBox": "1125,1036,36,28",
              "text": "ST"
            },
            {
              "boundingBox": "1185,1027,113,34",
              "text": "181740"
            }
          ]
        },
        {
          "boundingBox": "1124,1062,432,49",
          "words": [
            {
              "boundingBox": "1124,1071,252,40",
              "text": "UNIVERSALBORR"
            },
            {
              "boundingBox": "1400,1066,96,32",
              "text": "8X120"
            },
            {
              "boundingBox": "1519,1062,37,30",
              "text": "MM"
            }
          ]
        },
        {
          "boundingBox": "1123,1125,175,34",
          "words": [
            {
              "boundingBox": "1123,1129,36,30",
              "text": "ST"
            },
            {
              "boundingBox": "1183,1125,115,32",
              "text": "181738"
            }
          ]
        },
        {
          "boundingBox": "1122,1164,416,44",
          "words": [
            {
              "boundingBox": "1122,1170,255,38",
              "text": "UNIVERSRLBORR"
            },
            {
              "boundingBox": "1501,1164,37,31",
              "text": "MM"
            }
          ]
        },
        {
          "boundingBox": "1123,1225,170,33",
          "words": [
            {
              "boundingBox": "1123,1228,36,30",
              "text": "ST"
            },
            {
              "boundingBox": "1183,1225,110,32",
              "text": "316401"
            }
          ]
        },
        {
          "boundingBox": "1123,1270,355,39",
          "words": [
            {
              "boundingBox": "1123,1275,216,34",
              "text": "LÅSCYLINDER"
            },
            {
              "boundingBox": "1362,1270,116,33",
              "text": "2-PACK"
            }
          ]
        },
        {
          "boundingBox": "1123,1327,177,34",
          "words": [
            {
              "boundingBox": "1123,1330,37,31",
              "text": "ST"
            },
            {
              "boundingBox": "1183,1327,117,32",
              "text": "396026"
            }
          ]
        },
        {
          "boundingBox": "1124,1373,356,39",
          "words": [
            {
              "boundingBox": "1124,1377,216,35",
              "text": "LÅSCYLINDER"
            },
            {
              "boundingBox": "1363,1373,117,33",
              "text": "2-PRCK"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1644,820,118,524",
      "lines": [
        {
          "boundingBox": "1658,820,96,31",
          "words": [
            {
              "boundingBox": "1658,820,96,31",
              "text": "79,00"
            }
          ]
        },
        {
          "boundingBox": "1659,912,97,31",
          "words": [
            {
              "boundingBox": "1659,916,50,27",
              "text": "44,"
            },
            {
              "boundingBox": "1719,912,37,28",
              "text": "90"
            }
          ]
        },
        {
          "boundingBox": "1659,1004,98,32",
          "words": [
            {
              "boundingBox": "1659,1007,51,29",
              "text": "69,"
            },
            {
              "boundingBox": "1720,1004,37,28",
              "text": "90"
            }
          ]
        },
        {
          "boundingBox": "1661,1103,97,35",
          "words": [
            {
              "boundingBox": "1661,1103,97,35",
              "text": "49,90"
            }
          ]
        },
        {
          "boundingBox": "1644,1309,118,35",
          "words": [
            {
              "boundingBox": "1644,1309,118,35",
              "text": "299,00"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1064,1469,620,45",
      "lines": [
        {
          "boundingBox": "1064,1469,620,45",
          "words": [
            {
              "boundingBox": "1064,1481,237,33",
              "text": "-Rabattcheck"
            },
            {
              "boundingBox": "1324,1486,51,24",
              "text": "nr:"
            },
            {
              "boundingBox": "1384,1469,300,38",
              "text": "935011035567095"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1123,1584,159,82",
      "lines": [
        {
          "boundingBox": "1123,1584,159,33",
          "words": [
            {
              "boundingBox": "1123,1584,159,33",
              "text": "DELSUMMA"
            }
          ]
        },
        {
          "boundingBox": "1143,1635,116,31",
          "words": [
            {
              "boundingBox": "1143,1635,116,31",
              "text": "Rabatt"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1609,1570,180,189",
      "lines": [
        {
          "boundingBox": "1609,1570,160,36",
          "words": [
            {
              "boundingBox": "1609,1575,11,31",
              "text": "|"
            },
            {
              "boundingBox": "1648,1570,121,34",
              "text": "041,70"
            }
          ]
        },
        {
          "boundingBox": "1690,1621,99,34",
          "words": [
            {
              "boundingBox": "1690,1621,99,34",
              "text": "50,00"
            }
          ]
        },
        {
          "boundingBox": "1651,1725,120,34",
          "words": [
            {
              "boundingBox": "1651,1727,53,32",
              "text": "991"
            },
            {
              "boundingBox": "1715,1746,9,13",
              "text": ","
            },
            {
              "boundingBox": "1732,1725,39,32",
              "text": "70"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "992,1737,310,1226",
      "lines": [
        {
          "boundingBox": "1123,1737,179,35",
          "words": [
            {
              "boundingBox": "1123,1737,179,35",
              "text": "SLUTSUMMA"
            }
          ]
        },
        {
          "boundingBox": "1036,2756,227,35",
          "words": [
            {
              "boundingBox": "1036,2756,227,35",
              "text": "Totalbelopp"
            }
          ]
        },
        {
          "boundingBox": "1140,2811,124,37",
          "words": [
            {
              "boundingBox": "1140,2811,53,35",
              "text": "991"
            },
            {
              "boundingBox": "1207,2833,8,15",
              "text": "/"
            },
            {
              "boundingBox": "1225,2811,39,34",
              "text": "70"
            }
          ]
        },
        {
          "boundingBox": "992,2927,271,36",
          "words": [
            {
              "boundingBox": "992,2928,159,35",
              "text": "Säljare:"
            },
            {
              "boundingBox": "1182,2927,81,33",
              "text": "7688"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1330,2754,145,92",
      "lines": [
        {
          "boundingBox": "1330,2754,144,34",
          "words": [
            {
              "boundingBox": "1330,2754,39,33",
              "text": "Ex"
            },
            {
              "boundingBox": "1394,2754,80,34",
              "text": "Moms"
            }
          ]
        },
        {
          "boundingBox": "1352,2809,123,37",
          "words": [
            {
              "boundingBox": "1352,2809,123,37",
              "text": "793,36"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1563,2752,126,92",
      "lines": [
        {
          "boundingBox": "1563,2752,125,33",
          "words": [
            {
              "boundingBox": "1563,2752,82,33",
              "text": "Moms"
            },
            {
              "boundingBox": "1670,2755,18,27",
              "text": "%"
            }
          ]
        },
        {
          "boundingBox": "1586,2808,103,36",
          "words": [
            {
              "boundingBox": "1586,2808,103,36",
              "text": "25,00"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1780,2751,123,93",
      "lines": [
        {
          "boundingBox": "1820,2751,83,33",
          "words": [
            {
              "boundingBox": "1820,2751,83,33",
              "text": "Moms"
            }
          ]
        },
        {
          "boundingBox": "1780,2807,123,37",
          "words": [
            {
              "boundingBox": "1780,2807,123,37",
              "text": "198,34"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "985,2924,966,573",
      "lines": [
        {
          "boundingBox": "1523,2924,83,33",
          "words": [
            {
              "boundingBox": "1523,2924,83,33",
              "text": "7618"
            }
          ]
        },
        {
          "boundingBox": "1288,2926,167,33",
          "words": [
            {
              "boundingBox": "1288,2939,17,7",
              "text": "-"
            },
            {
              "boundingBox": "1330,2926,125,33",
              "text": "Sabina"
            }
          ]
        },
        {
          "boundingBox": "1182,2981,468,36",
          "words": [
            {
              "boundingBox": "1182,2983,38,34",
              "text": "24"
            },
            {
              "boundingBox": "1245,2982,146,34",
              "text": "oktober"
            },
            {
              "boundingBox": "1416,2982,82,34",
              "text": "2016"
            },
            {
              "boundingBox": "1547,2982,10,33",
              "text": "1"
            },
            {
              "boundingBox": "1571,2981,79,34",
              "text": "7:20"
            }
          ]
        },
        {
          "boundingBox": "991,2985,103,33",
          "words": [
            {
              "boundingBox": "991,2985,103,33",
              "text": "Datum"
            }
          ]
        },
        {
          "boundingBox": "1161,3040,403,34",
          "words": [
            {
              "boundingBox": "1161,3040,96,34",
              "text": "44601"
            },
            {
              "boundingBox": "1288,3040,140,34",
              "text": "Kvitto:"
            },
            {
              "boundingBox": "1460,3040,104,34",
              "text": "51756"
            }
          ]
        },
        {
          "boundingBox": "990,3042,103,33",
          "words": [
            {
              "boundingBox": "990,3042,103,33",
              "text": "Kassa"
            }
          ]
        },
        {
          "boundingBox": "1096,3157,728,40",
          "words": [
            {
              "boundingBox": "1096,3159,105,38",
              "text": "Spara"
            },
            {
              "boundingBox": "1225,3157,163,39",
              "text": "kvittot,"
            },
            {
              "boundingBox": "1418,3157,127,39",
              "text": "gäller"
            },
            {
              "boundingBox": "1570,3169,63,26",
              "text": "som"
            },
            {
              "boundingBox": "1657,3158,167,39",
              "text": "garanti."
            }
          ]
        },
        {
          "boundingBox": "1268,3217,388,39",
          "words": [
            {
              "boundingBox": "1268,3217,103,39",
              "text": "Öppet"
            },
            {
              "boundingBox": "1397,3218,62,38",
              "text": "köp"
            },
            {
              "boundingBox": "1484,3218,41,37",
              "text": "30"
            },
            {
              "boundingBox": "1550,3218,106,38",
              "text": "dager"
            }
          ]
        },
        {
          "boundingBox": "1290,3276,317,39",
          "words": [
            {
              "boundingBox": "1290,3276,192,38",
              "text": "VÄLKOMMEN"
            },
            {
              "boundingBox": "1506,3278,101,37",
              "text": "ÅTER!"
            }
          ]
        },
        {
          "boundingBox": "1116,3335,719,42",
          "words": [
            {
              "boundingBox": "1116,3337,41,36",
              "text": "Om"
            },
            {
              "boundingBox": "1182,3335,82,38",
              "text": "ditt"
            },
            {
              "boundingBox": "1290,3346,84,28",
              "text": "namn"
            },
            {
              "boundingBox": "1398,3337,63,38",
              "text": "och"
            },
            {
              "boundingBox": "1485,3349,261,28",
              "text": "personnummer"
            },
            {
              "boundingBox": "1771,3338,64,37",
              "text": "har"
            }
          ]
        },
        {
          "boundingBox": "1032,3395,894,42",
          "words": [
            {
              "boundingBox": "1032,3397,146,36",
              "text": "lämnats"
            },
            {
              "boundingBox": "1204,3395,62,38",
              "text": "för"
            },
            {
              "boundingBox": "1290,3395,61,38",
              "text": "att"
            },
            {
              "boundingBox": "1377,3399,194,36",
              "text": "genomföra"
            },
            {
              "boundingBox": "1596,3399,61,36",
              "text": "ett"
            },
            {
              "boundingBox": "1685,3399,241,38",
              "text": "JulaPro-köp"
            }
          ]
        },
        {
          "boundingBox": "985,3455,966,42",
          "words": [
            {
              "boundingBox": "985,3456,193,37",
              "text": "behandlar"
            },
            {
              "boundingBox": "1203,3455,85,37",
              "text": "Jula"
            },
            {
              "boundingBox": "1312,3456,84,37",
              "text": "dina"
            },
            {
              "boundingBox": "1421,3458,195,39",
              "text": "uppgifter"
            },
            {
              "boundingBox": "1645,3462,12,33",
              "text": "i"
            },
            {
              "boundingBox": "1686,3458,173,38",
              "text": "enlighet"
            },
            {
              "boundingBox": "1886,3461,65,36",
              "text": "med"
            }
          ]
        }
      ]
    }
  ]
}

Confectioner answered 28/11, 2016 at 4:6 Comment(3)

Did you find solution? – Calve 27/2, 2017 at 12:57

We extract all the words putting them in a single list. We will later use Azure machine learning to determine what belongs together and what kind of property it is. But for now we just use the bounding boxes for the separate words to create clickable areas where the user needs to select the right boxes for different properties. – Confectioner 28/2, 2017 at 7:40

which machine learning algorithm did you use? @Confectioner – Thorne 23/1, 2020 at 12:21

R

5

This is not computer vision problem, its NLP/text pattern recognition problem. In other words, none of the OCRs would do what you want to do; they only extracts text from images.

The usual approach is to collect many different types of receipts, study their structure and then either use rule based approach or machine learning based approach to classify each piece of information. The classifier can have categories like { ItemName, ItemPrice, Subtotal, Total, Heading, Other }. You can use the bounding boxes to form grid cells and use the neighbouring cells as your feature.

Tutorial

Applying OCR Technology for Receipt Recognition.

Cloud Services

Open source project

Rapp answered 3/12, 2016 at 6:29 Comment(3)

We are planning to use Microsofts Computer Vision together with Azure Machine Learning. Do you if there are any modules within Azure ML to accomplish this? – Confectioner 4/12, 2016 at 17:49

@ShitalShah It would seem the link to the tutorial is not existant anymore. Can you update it if it still exists? – Makeshift 17/10, 2018 at 14:56

Updated tutorial link, also added bunch of available cloud services – Rapp 17/10, 2018 at 23:1

I

0

Create model

public class Region
{
    public string BoundingBox { get; set; }
    public List<Line> Lines { get; set; }
}

public class Line
{
    public string BoundingBox { get; set; }
    public List<Word> Words { get; set; }
}

public class Word
{
    public string BoundingBox { get; set; }
    public string Text { get; set; }
}

If you want C# types for the returned response, you can use the official client SDK in github. It's also available in NuGet.

Once you have the OcrResults, and you just want the text, you could write some hacky C# code with Linq like this:

string OcrResultsToString(OcrResult result)
{
    return string.Join("\n",
        result.Regions.ToList().Select(region =>
            string.Join(" ", region.Lines.ToList().Select(line =>
                 string.Join(" ", line.Words.ToList().Select(word =>
                     word.Text).ToArray())).ToArray())).ToArray());
}

Or, if that hurts your eyes, you could use conventional loops like this:

 string OcrResultsToString(OcrResults results)
 {
    StringBuilder stringBuilder = new StringBuilder();

    if (results != null && results.Regions != null)
    {
        foreach (var item in results.Regions)
        {
            foreach (var line in item.Lines)
            {
                foreach (var word in line.Words)
                {
                    stringBuilder.Append(word.Text);
                    stringBuilder.Append(" ");
                }
                stringBuilder.AppendLine();
            }
            stringBuilder.AppendLine();
        }
    }
    return stringBuilder.ToString();
}

Inkling answered 28/6, 2018 at 6:23 Comment(1)

This is just a way to create a string of the result and not a solution to our problem. The problem we have is that the data we get is not ordered such that we can use i as receipt data. But this is as earler mentioned a ML problem – Confectioner 14/12, 2018 at 6:34

Recommended topics

Hot tags