Mobile Application for Receipt Fraud Detection Based on Optical Character Recognition

Ich schrieb hier über eine Situation, die ich in einem Supermarkt hatte, als der Preis am Regal anders war als der in ihrem System. Hier habe ich auch über die ICMV (International Conference on Machine Vision) 2019 Konferenz geschrieben, wo ich zusammen mit diesem Paper auch unser Paper „Mobile Application for Receipt Fraud Detection Based on Optical Character Recognition“ vorgestellt wurde. Ich habe die Arbeit in der mündlichen Sitzung A-2 „Target Detection and Tracking“ vorgestellt.

Abstrakt: Dieses Papier stellt ein Verfahren zur Erkennung von Quittungsbetrug durch die Implementierung eines Object Character Recognition (OCR)-Algorithmus vor, der aus Bildverarbeitungstechniken und Convolutional Neural Networks (CNNs) besteht. Wir haben zwei CNN-Modelle in eine Smartphone-Anwendung implementiert, die es den Kunden ermöglicht, Fotos von Produkten, die sie kaufen möchten (auch um ihre Preisschilder zu schneiden), während sie in einem Hypermarkt/Supermarkt vorhanden sind, sowie von der bezahlten Quittung zu machen, und es gelingt uns, automatisch alle Preise (mehrere Ziffern einschließlich Dezimalstellen) der im Regal gesehenen Produkte und alle Preise, die in der bezahlten Quittung enthalten sind, die von der Kasse erhalten wurden, zu identifizieren und zu vergleichen. Diese Anwendung hilft dem Kunden, einen Quittungsbetrug aufgrund eines Computer- oder menschlichen Fehlers auf kostengünstige und bequeme Weise zu erkennen. Experimentelle Ergebnisse zeigen eine Gesamttestgenauigkeit von 99,96% für das CNN, das für die Identifizierung der Produktpreise verantwortlich ist, und eine Gesamttestgenauigkeit von 99,35% für das CNN, das für die Identifizierung der Empfangspreise verantwortlich ist.

Sie können den Artikel hier lesen.

Hier habe ich alle Projektdateien (2 Supermarkt Prices/Digits Datasets + 2 Product Prices and Receipt Prices DL models + Android Mobile App Dateien) bezüglich unserer mobilen Anwendung zur Aufdeckung von Quittungsbetrug in einem Supermarkt veröffentlicht.

NeuralNetworks.py

„““
Code created by Sorin Liviu Jurj for the paper called „Mobile Application for Receipt Fraud Detection Based on Optical Character Recognition“.
More information about the paper can be found here: https://www.jurj.de/mobile-application-for-receipt-fraud-detection-based-on-optical-character-recognition/
„““
import numpy as np
import matplotlib.pyplot as plt
import cv2
import os
import keras
from tqdm import tqdm
import tensorflow as tf

def box2rects(contours):
„““
Return the bounding rectangles of the input countours
„““
return [cv2.boundingRect(c) for c in contours]

def extractCharacters(img):
„““
Extract digits from the input image and returns the image and positions of each individual digit
„““
original = img.copy()
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
height , width = img.shape
img = cv2.bitwise_not(img)
blurSize = 2*int(height*0.01/2)+1
img = cv2.GaussianBlur(img, (blurSize, blurSize), 0)
img = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 2*int(height*0.9/2)+1, -20)
contours, _ = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
splits = []
for x, y, w, h in box2rects(contours):
if h>height*0.05 and w>width*0.02 and h<height*0.5 and w<width*0.2:
split = original[max(0,y-10):y+h+20,max(0,x-10):x+w+20]
splits.append((split, (x, y, w, h)))
return splits

def cropped2digits(destination):
„““
Takes cropped images located at data1/cropped and saves the extracted digits at the input destination
„““
for fil in tqdm(os.listdir(‚data1/cropped‘)):
pixels = cv2.imread(os.path.join(‚data1/cropped‘, fil))
splits = extractCharacters(pixels)
for split in splits:
split = split[0]
cv2.imwrite(os.path.join(destination, str(np.random.uniform(0,1000))+“.jpg“), split)

def rotateRandomly(img):
„““
Picks an angle between -10 and 10 and rotates the image by that angle
„““
rows, cols = img.shape[0], img.shape[1]
rotationAngle = np.random.uniform(-10, 10)
# center of rotation, rotation angle, scale
M = cv2.getRotationMatrix2D((cols/2, rows/2), rotationAngle, 0.9)
return cv2.warpAffine(img,M,(cols,rows), borderMode=cv2.BORDER_REPLICATE)

def addShadow(img):
„““
Chooses a random straigth line and shades one side of the image by a random amount
„““
y1, y2 = np.random.randint(0, img.shape[0], 2)
x1, x2 = np.random.randint(0, img.shape[1], 2)
img = img.copy()
if x1==x2:
m = img.shape[0]*(y2-y1)
else:
m = (y2-y1)/(x2-x1)
shade = np.random.randint(0, 30)
sign = np.random.choice([-1,1])
for y in range(img.shape[0]):
for x in range(img.shape[1]):
if y*sign > (m*x – m*x1 + y1)*sign:
img[y][x] = img[y][x] – min(shade, img[y][x][0], img[y][x][1], img[y][x][2])
return img

def randomDownWarp(img):
„““
Warps the image randomly by enlarging the bottom side of the image
„““
img = img.copy()
height , width, _ = img.shape
pad = np.random.randint(0, int(width*0.15))
img = cv2.copyMakeBorder(img, 0, 0, pad, pad, cv2.BORDER_REPLICATE)
finalH, finalW, _ = img.shape
warpPoints = np.float32([ [0, 0], [pad, finalH], [finalW, 0], [finalW-pad, finalH] ])
warpDest = np.float32([[0, 0], [0, finalH], [finalW, 0], [finalW, finalH]])
M = cv2.getPerspectiveTransform(warpPoints, warpDest)
img = cv2.warpPerspective(img, M, (finalW, finalH), borderMode=cv2.BORDER_REPLICATE )
return img

def randomTopWarp(img):
„““
Warps the image randomly by enlarging the top side of the image
„““
img = img.copy()
height , width, _ = img.shape
pad = np.random.randint(0, int(width*0.15))
img = cv2.copyMakeBorder(img, 0, 0, pad, pad, cv2.BORDER_REPLICATE)
finalH, finalW, _ = img.shape
warpPoints = np.float32([ [pad, 0], [0, finalH], [finalW-pad, 0], [finalW, finalH] ])
warpDest = np.float32([[0, 0], [0, finalH], [finalW, 0], [finalW, finalH]])
M = cv2.getPerspectiveTransform(warpPoints, warpDest)
img = cv2.warpPerspective(img, M, (finalW, finalH), borderMode=cv2.BORDER_REPLICATE )
return img

def randomLeftWarp(img):
„““
Warps the image randomly by enlarging the left side of the image
„““
img = img.copy()
height , width, _ = img.shape
pad = np.random.randint(0, int(height*0.15))
img = cv2.copyMakeBorder(img, pad, pad, 0, 0, cv2.BORDER_REPLICATE)
finalH, finalW, _ = img.shape
warpPoints = np.float32([ [0, pad], [0, finalH-pad], [finalW, 0], [finalW, finalH] ])
warpDest = np.float32([[0, 0], [0, finalH], [finalW, 0], [finalW, finalH]])
M = cv2.getPerspectiveTransform(warpPoints, warpDest)
img = cv2.warpPerspective(img, M, (finalW, finalH), borderMode=cv2.BORDER_REPLICATE )
return img

def randomRightWarp(img):
„““
Warps the image randomly by enlarging the right side of the image
„““
img = img.copy()
height , width, _ = img.shape
pad = np.random.randint(0, int(height*0.15))
img = cv2.copyMakeBorder(img, pad, pad, 0, 0, cv2.BORDER_REPLICATE)
finalH, finalW, _ = img.shape
warpPoints = np.float32([ [0, 0], [0, finalH], [finalW, pad], [finalW, finalH-pad] ])
warpDest = np.float32([[0, 0], [0, finalH], [finalW, 0], [finalW, finalH]])
M = cv2.getPerspectiveTransform(warpPoints, warpDest)
img = cv2.warpPerspective(img, M, (finalW, finalH), borderMode=cv2.BORDER_REPLICATE )
return img

def randomWarp(img):
„““
Picks a random horizontal warp and a random vertical warp and applies both to the image
„““
horizontalWarp = np.random.choice([randomLeftWarp, randomRightWarp])
verticalWarp = np.random.choice([randomDownWarp, randomTopWarp])
return verticalWarp(horizontalWarp(img))

def addRandomPadding(img):
„““
Pads the image from each side by a random amount
„““
height, width, _ = img.shape
pad1 = np.random.randint(1, int(width*0.2))
pad2 = np.random.randint(1, int(width*0.2))
pad3 = np.random.randint(1, int(height*0.2))
pad4 = np.random.randint(1, int(height*0.2))
return cv2.copyMakeBorder(img, pad3, pad4, pad1, pad2, cv2.BORDER_REPLICATE)

def addPadding(img, pad1, pad2, pad3, pad4):
„““
Add a padding to the input image according to the specified amounts
„““
return cv2.copyMakeBorder(img, pad3, pad4, pad1, pad2, cv2.BORDER_REPLICATE)

def pad_and_resize(img, height, width):
„““
Rezises and pad the image in order to return an image of the target height and width
„““
img = img.copy()
desiredRatio = height/width
ratio = img.shape[0]/img.shape[1]
if ratio == desiredRatio:
return cv2.resize(img, dsize=(width, height))
elif ratio > desiredRatio:
img = cv2.resize(img, dsize=(max(1, int(height*img.shape[1]/img.shape[0])), height))
delta_w = width – img.shape[1]
img = cv2.copyMakeBorder(img, 0, 0, delta_w//2, delta_w-delta_w//2, cv2.BORDER_CONSTANT, value=[255, 255, 255])
assert img.shape[0] == height
assert img.shape[1] == width
return img
else:
img = cv2.resize(img, dsize=(width, max(1,int(width*img.shape[0]/img.shape[1]))))
delta_h = height – img.shape[0]
img = cv2.copyMakeBorder(img, delta_h//2, delta_h-delta_h//2, 0, 0, cv2.BORDER_CONSTANT, value=[255, 255, 255])
assert img.shape[0] == height
assert img.shape[1] == width
return img

def augmentThisData(data):
„““
Applies the four random transformations to generate a new augmented sample
„““
return addRandomPadding(rotateRandomly(addShadow(randomWarp(data))))

def dataAugmentation(source, destination):
„““
From the source images augment the data into the destination
„““
for digit in tqdm(os.listdir(source)):
files = os.listdir(os.path.join(source, digit))
for iteration in tqdm(range(1000)):
originalImage = cv2.imread(os.path.join(source, digit, np.random.choice(files)))
generatedImage = pad_and_resize(augmentThisData(originalImage), 100, 50)
cv2.imwrite(os.path.join(destination, digit, „{}_{}.jpg“.format(digit,iteration)), generatedImage)

def confusionMatrix(ytrue, ypred):
„““
Confusion Matrix building function
„““
ytrue = tf.argmax(ytrue, axis=1)
ypred = tf.argmax(ypred, axis=1)
return tf.confusion_matrix(ytrue, ypred, num_classes=12)

def recall(M, i):
„““
Recall function
„““
return M[i][i]/float((tf.reduce_sum(M, axis=0)[i]).eval())

def precision(M, i):
„““
Precision function
„““
return M[i][i]/float((tf.reduce_sum(M, axis=1)[i]).eval())

def F1Score(M, i):
„““
F1Score function
„““
prec = precision(M, i)
recal = recall(M, i)
return 2*precision(M, i)*recall(M, i)/(precision(M, i)+recall(M, i))

def shuffle_in_unison(a, b):
„““
Shuffles a and b with the same permutations
„““
assert len(a) == len(b)
shuffled_a = np.empty(a.shape, dtype=a.dtype)
shuffled_b = np.empty(b.shape, dtype=b.dtype)
permutation = np.random.permutation(len(a))
for old_index, new_index in enumerate(permutation):
shuffled_a[new_index] = a[old_index]
shuffled_b[new_index] = b[old_index]
return shuffled_a, shuffled_b

def digitRecognizer():
„““
Digit recognizer CNN model for the first algorithm
„““
model = keras.models.Sequential()
model.add(keras.layers.Conv2D(4, [10, 10], input_shape=[100, 50, 1], strides=(2, 2), dilation_rate=(1, 1), activation=’relu‘))
model.add(keras.layers.Dropout(0.5))
model.add(keras.layers.MaxPooling2D(pool_size=(2, 2), strides=None, padding=’valid‘, data_format=None))
model.add(keras.layers.Flatten())
model.add(keras.layers.Dense(12, activation=“relu“))
model.add(keras.layers.Dense(12, activation=“softmax“))
model.compile(optimizer=’rmsprop‘, loss=’categorical_crossentropy‘, metrics=[‚accuracy‘])
return model

def plotHistory(history,name):
„““
Plots a given training history from a keras model
„““
acc = history.history[‚acc‘]
val_acc = history.history[‚val_acc‘]
loss = history.history[‚loss‘]
val_loss = history.history[‚val_loss‘]
epochs = range(1, len(acc)+1)
with open(„results_{}.csv“.format(name), ‚w‘) as w:
w.write(„Training accuracy,Validation accuracy,Training loss,Validation loss\n“)
for i in range(1, len(acc)):
w.write(„{},{},{},{}\n“.format(„%.4f“ % acc[i], „%.4f“ % val_acc[i], „%.4f“ % loss[i], „%.4f“ % val_loss[i]))
plt.semilogx(epochs, acc, ‚bo‘, color=’green‘, label=’Training acc‘)
plt.semilogx(epochs, val_acc, ‚b‘,color=’blue‘,label=’Validation acc‘)
plt.xlabel(„Epoch“)
plt.ylabel(„Accuracy“)
plt.title(‚Training and validation accuracy‘)
plt.legend()
plt.figure()
plt.semilogx(epochs, loss, ‚bo‘,color=’green‘, label=’Training loss‘)
plt.semilogx(epochs, val_loss, ‚b‘,color=’blue‘, label=’Validation loss‘)
plt.xlabel(„Epoch“)
plt.ylabel(„Loss“)
plt.title(‚Training and validation loss‘)
plt.legend()
plt.show()

def getTestData(dataLocation):
„““
Loads test data from dataLocation and returns images and labels
„““
targetHeight=100
targetWidth=50
labelsDict = {„zero“:0, „one“:1, „two“:2, „three“:3, „four“:4, „five“:5, „six“:6, „seven“:7, „eight“:8, „nine“:9, „comma“: 10, „none“:11}
digits = os.listdir(dataLocation)
images = []
labels = []
for digit in digits:
selImages = os.listdir(os.path.join(dataLocation, digit))
for chosenImage in selImages:
img = cv2.imread(os.path.join(dataLocation, digit, chosenImage))
height, width, _ = img.shape
img = addPadding(img, int(width*0.1), int(width*0.1), int(height*0.1), int(height*0.1))
img = pad_and_resize(img, targetHeight, targetWidth)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img = np.array(img).reshape([targetHeight, targetWidth, 1])
img = np.array(img, dtype=np.float64)
mean = np.mean(img)
std = np.std(img)
img -= mean
img /= std
images.append(img)
labels.append(digit)
images = np.array(images)
labels = [labelsDict[label] for label in labels]
labels = np.array(labels)
return images, labels

def getTestReceiptData(dataLocation):
„““
Loads test receipt data from dataLocation and returns images and labels
„““
height=20
width=10
labelsDict = {„zero“:0, „one“:1, „two“:2, „three“:3, „four“:4, „five“:5, „six“:6, „seven“:7, „eight“:8, „nine“:9, „comma“: 10, „none“:11}
digits = os.listdir(dataLocation)
images = []
labels = []
for digit in digits:
selImages = os.listdir(os.path.join(dataLocation, digit))
for chosenImage in selImages:
img = cv2.imread(os.path.join(dataLocation, digit, chosenImage))
img = pad_and_resize(img, height, width)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img = np.array(img).reshape([height, width, 1])
img = np.array(img, dtype=np.float64)
mean = np.mean(img)
std = np.std(img)
img -= mean
img /= std
images.append(img)
labels.append(digit)
images = np.array(images)
labels = [labelsDict[label] for label in labels]
labels = np.array(labels)
return images, labels

def train(source, destination,name=“digits“):
„““
Trains and saves the model with data from given source
„““
model = digitRecognizer()
images = []
labels = []
labelsDict = {„zero“:0, „one“:1, „two“:2, „three“:3, „four“:4, „five“:5, „six“:6, „seven“:7, „eight“:8, „nine“:9, „comma“: 10, „none“:11}
for digit in os.listdir(source):
label = digit
for image in os.listdir(os.path.join(source, digit)):
img = cv2.imread(os.path.join(source, digit, image))
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img = np.array(img).reshape([100, 50, 1])
img = np.array(img, dtype=np.float64)
mean = np.mean(img)
std = np.std(img)
img -= mean
img /= std
images.append(img)
labels.append(labelsDict[label])
one_hot_labels = np.zeros((len(labels), 12))
one_hot_labels[np.arange(len(labels)), labels] = 1
one_hot_labels = np.array(one_hot_labels)
images = np.array(images)
images, one_hot_labels = shuffle_in_unison(images, one_hot_labels)
history = model.fit(x=images, y=one_hot_labels, shuffle=True, batch_size=32, epochs=30, validation_split=0.1, callbacks=[keras.callbacks.TerminateOnNaN()])
scores = model.evaluate(images,one_hot_labels,verbose=0)
print(„%s: %.2f%%“ % (model.metrics_names[1],scores[1]*100))
model.save_weights(destination)
model.save(„digit_model.h5“)
plotHistory(history,name)

def test(weightLocation, dataLocation):
„““
Tests the model saved at weightLocation with data at dataLocation
„““
model = digitRecognizer()
model.load_weights(weightLocation)
images, labels = getTestData(dataLocation)
predictions = model.predict_on_batch(images)
one_hot_labels = np.zeros((len(labels), 12))
one_hot_labels[np.arange(len(labels)), labels] = 1
M = confusionMatrix(one_hot_labels, predictions)
classes = list(range(10))+[„Comma“, „Noise“]
with open(„results_digits_scores.csv“, ‚w‘) as w:
with tf.Session().as_default():
suma = 0
M = M.eval()
w.write(„Digit,Test accuracy,Samples,Precision,Recall,F1Score\n“)
total = np.sum(M)
for i in range(12):
a=M[i][i]
x=np.sum(M,axis=0)[i]
y=np.sum(M,axis=1)[i]
accuracy = float((total + 2*a – x – y)/float(total))
suma += accuracy
w.write(„{},{},{},{},{},{}\n“.format(classes[i],“%.4f“ % accuracy, np.sum(M, axis=1)[i], „%.4f“ % precision(M, i), „%.4f“ % recall(M, i), „%.4f“ % F1Score(M, i)))
suma /= 12
w.write(„{},{}“.format(„Overall accuracy“,suma))
w.write(„{},{}“.format(„Accuracy from confusion matrix“,accuracyMat(M)))

def testReceipt(weightLocation, dataLocation,model):
„““
Tests the receipt digit model saved at weightLocation with data an dataLocation
„““
model.load_weights(weightLocation)

images, labels = getTestReceiptData(dataLocation)

predictions = model.predict_on_batch(images)
one_hot_labels = np.zeros((len(labels), 12))
one_hot_labels[np.arange(len(labels)), labels] = 1
M = confusionMatrix(one_hot_labels, predictions)
classes = list(range(10))+[„Comma“, „Noise“]
with open(„results_receipt_scores.csv“, ‚w‘) as w:
with tf.Session().as_default():
suma = 0
M = M.eval()
print(„Confusion Matrix“)
print(M)
w.write(„Digit,Test accuracy,Samples,Precision,Recall,F1Score\n“)
total = np.sum(M)
for i in range(12):
a=M[i][i]
x=np.sum(M,axis=0)[i]
y=np.sum(M,axis=1)[i]
accuracy = float((total + 2*a – x – y)/float(total))
suma += accuracy
w.write(„{},{},{},{},{},{}\n“.format(classes[i],“%.4f“ % accuracy, np.sum(M, axis=1)[i], „%.4f“ % precision(M, i), „%.4f“ % recall(M, i), „%.4f“ % F1Score(M, i)))
suma /= 12
w.write(„{},{}“.format(„Overall accuracy“,suma))
w.write(„{},{}“.format(„Accuracy from confusion matrix“,accuracyMat(M)))

def show(img):
„““
Function for showing images if cv2.imshow doesn’t work
„““
plt.imshow(img, cmap = ‚gray‘, interpolation = ‚bicubic‘)
plt.xticks([]), plt.yticks([]) # to hide tick values on X and Y axis
plt.show()

def receiptDigitExtraction(img):
„““
Extract digits from the input image and returns the image and positions of each individual digit
„““
original = img.copy()
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
height , width = img.shape
img = cv2.bitwise_not(img)
blurSize = 2*int(width*0.01/2)+1
img = cv2.GaussianBlur(img, (blurSize, blurSize), 0)
img = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 2*int(width*0.7/2)+1, -20)
_, contours, _ = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
splits = []
for x, y, w, h in box2rects(contours):
if h>height*0.05 and w>width*0.02:
split = original[max(0,y-2):y+h+4,max(0,x-2):x+w+4]
splits.append((split, (x, y, w, h)))

return splits

def randomLightLevel(img):
„““
Randomly changes the brightness of the image
„““
return cv2.add(img, np.array([np.random.uniform(-50, 50)]))

def receiptDataAugmentation(source, destination):
„““
From the source receipt digits images from receipt augment the data into the destination
„““
for digit in tqdm(os.listdir(source)):
files = os.listdir(os.path.join(source, digit))
for iteration in tqdm(range(100)):
originalImage = cv2.imread(os.path.join(source, digit, np.random.choice(files)))
generatedImage = pad_and_resize(augmentThisData(originalImage), 20, 10)
cv2.imwrite(os.path.join(destination, digit, „{}_{}.jpg“.format(digit,iteration)), generatedImage)

def receiptDigitRecognizer():
„““
Digit recognizer CNN model for the receipt digits
„““
model = keras.models.Sequential()
model.add(keras.layers.Conv2D(20, [2, 2], input_shape=[20, 10, 1], strides=(1, 1), padding=’valid‘, data_format=None, dilation_rate=(1, 1), activation=’relu‘))
model.add(keras.layers.MaxPooling2D(pool_size=(2, 2), strides=None, padding=’valid‘, data_format=None))
model.add(keras.layers.Dropout(0.5))
model.add(keras.layers.Flatten())
model.add(keras.layers.Dense(20, activation=“relu“))
model.add(keras.layers.Dense(12, activation=“softmax“))
model.compile(optimizer=’rmsprop‘, loss=’categorical_crossentropy‘, metrics=[‚accuracy‘])
return model

def trainReceipt(source, model, destination,name=“receipt“):
„““
Trains and saves the receipt digit model with data from given source
„““
images = []
labels = []
labelsDict = {„zero“:0, „one“:1, „two“:2, „three“:3, „four“:4, „five“:5, „six“:6, „seven“:7, „eight“:8, „nine“:9, „comma“: 10, „none“:11}
for digit in os.listdir(source):
label = digit
for image in os.listdir(os.path.join(source, digit)):
img = cv2.imread(os.path.join(source, digit, image))
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img = cv2.resize(img,(20,10))
img = np.array(img).reshape([20, 10, 1])
img = np.array(img, dtype=np.float64)
mean = np.mean(img)
std = np.std(img)
img -= mean
img /= std
images.append(img)
labels.append(labelsDict[label])
one_hot_labels = np.zeros((len(labels), 12))
one_hot_labels[np.arange(len(labels)), labels] = 1
one_hot_labels = np.array(one_hot_labels)
images = np.array(images)
images, one_hot_labels = shuffle_in_unison(images, one_hot_labels)
history = model.fit(x=images, y=one_hot_labels, shuffle=True, batch_size=1000, epochs=1100, validation_split=0.1, callbacks=[keras.callbacks.TerminateOnNaN()])
scores = model.evaluate(images,one_hot_labels,verbose=0)
print(„Model metric %s: %.2f%%“ % (model.metrics_names[1],scores[1]*100))
model.save_weights(destination)
model.save(„receipt_model.h5“)
plotHistory(history,name)

def train_test_CNN1():
„““
Training and testing the first model
„““
model = digitRecognizer()
model.summary()
cropped2digits(„data/digitsFinal“)
dataAugmentation(„data/digitsFinal“, „data/augmentedFinal“)
weightLocation = „digit_weights.h5“
dataLocation = „data/augmentedFinal“
testLocation = „data/digitsFinal“
train(dataLocation,weightLocation)
test(weightLocation, testLocation)

def train_test_CNN2():
„““
Training and testing the second model
„““
receiptDataAugmentation(‚receiptDigits‘,’augmentedReceipt‘)
model = receiptDigitRecognizer()
model.compile(optimizer=’rmsprop‘, loss=’categorical_crossentropy‘, metrics=[‚accuracy‘])
model.summary()
weightLocation = ‚receipt_weights.h5‘
dataLocation = ‚receiptData/augmentedReceipt‘
testLocation = ‚receiptData/receiptDigits‘
trainReceipt(dataLocation,model,weightLocation)
testReceipt(weightLocation, testLocation,model)

def pickle_CNN1():
„““
Serializing the first model’s weights
„““
model = digitRecognizer()
weightLocation = ‚digit_weights.h5‘
model.load_weights(weightLocation)
layers = model.layers
CNN1_conv2d_weights = layers[0].get_weights()[0]
CNN1_conv2d_bias = layers[0].get_weights()[1]
CNN1_dense1_weights = layers[4].get_weights()[0]
CNN1_dense1_bias = layers[4].get_weights()[1]
CNN1_dense2_weights = layers[5].get_weights()[0]
CNN1_dense2_bias = layers[5].get_weights()[1]
fnames = {„CNN1_conv2d_weights“:CNN1_conv2d_weights,“CNN1_conv2d_bias“:CNN1_conv2d_bias,
„CNN1_dense1_weights“:CNN1_dense1_weights,“CNN1_dense1_bias“:CNN1_dense1_bias,
„CNN1_dense2_weights“:CNN1_dense2_weights,“CNN1_dense2_bias“:CNN1_dense2_bias}
for fname in fnames.keys():
with open(„{}.pickle“.format(fname),“wb“) as f:
pickle.dump(fnames[fname],f)

def pickle_CNN2():
„““
Serializing the second model’s weights
„““
model = receiptDigitRecognizer()
weightLocation = ‚receipt_weights.h5‘
model.load_weights(weightLocation)
layers = model.layers
CNN2_conv2d_weights = layers[0].get_weights()[0]
CNN2_conv2d_bias = layers[0].get_weights()[1]
CNN2_dense1_weights = layers[4].get_weights()[0]
CNN2_dense1_bias = layers[4].get_weights()[1]
CNN2_dense2_weights = layers[5].get_weights()[0]
CNN2_dense2_bias = layers[5].get_weights()[1]
fnames = {„CNN2_conv2d_weights“:CNN2_conv2d_weights,“CNN2_conv2d_bias“:CNN2_conv2d_bias,
„CNN2_dense1_weights“:CNN2_dense1_weights,“CNN2_dense1_bias“:CNN2_dense1_bias,
„CNN2_dense2_weights“:CNN2_dense2_weights,“CNN2_dense2_bias“:CNN2_dense2_bias}
for fname in fnames.keys():
with open(„{}.pickle“.format(fname),“wb“) as f:
pickle.dump(fnames[fname],f)

if __name__ == „__main__“:
train_test_CNN1()
pickle_CNN1()
train_test_CNN2()
pickle_CNN2()

Testing_model.py

„““
Code created by Sorin Liviu Jurj for the paper called „Mobile Application for Receipt Fraud Detection Based on Optical Character Recognition“.
More information about the paper can be found here: https://www.jurj.de/mobile-application-for-receipt-fraud-detection-based-on-optical-character-recognition/
„““
import cv2
import time
import os
import keras
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

def box2rects(contours):
„““
Return the bounding rectangles of the input countours
„““
return [cv2.boundingRect(c) for c in contours]

def addPadding(img, pad1, pad2, pad3, pad4):
„““
Add a padding to the input image according to the specified amounts
„““
return cv2.copyMakeBorder(img, pad3, pad4, pad1, pad2, cv2.BORDER_REPLICATE)

def pad_and_resize(img, height, width):
„““
Rezises and pad the image in order to return an image of the target height and width
„““
img = img.copy()
desiredRatio = height/width
ratio = img.shape[0]/img.shape[1]
if ratio == desiredRatio:
return cv2.resize(img, dsize=(width, height))
elif ratio > desiredRatio:
img = cv2.resize(img, dsize=(max(1, int(height*img.shape[1]/img.shape[0])), height))
delta_w = width – img.shape[1]
img = cv2.copyMakeBorder(img, 0, 0, delta_w//2, delta_w-delta_w//2,
cv2.BORDER_CONSTANT, value=[255, 255, 255])
assert img.shape[0] == height
assert img.shape[1] == width
return img
else:
img = cv2.resize(img, dsize=(width, max(1,int(width*img.shape[0]/img.shape[1]))))
delta_h = height – img.shape[0]
img = cv2.copyMakeBorder(img, delta_h//2, delta_h-delta_h//2, 0, 0,
cv2.BORDER_CONSTANT, value=[255, 255, 255])
assert img.shape[0] == height
assert img.shape[1] == width
return img

def digitRecognizer():
„““
Digit recognizer CNN model
„““
model = keras.models.Sequential()
model.add(keras.layers.Conv2D(4, [10, 10], input_shape=[100, 50, 1], strides=(2, 2),
padding=’valid‘, data_format=None, dilation_rate=(1, 1), activation=’relu‘,
use_bias=True, kernel_initializer=’glorot_uniform‘, bias_initializer=’zeros‘,
kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None,
kernel_constraint=None, bias_constraint=None))
model.add(keras.layers.Dropout(0.5))
model.add(keras.layers.MaxPooling2D(pool_size=(2, 2), strides=None, padding=’valid‘,
data_format=None))
model.add(keras.layers.Flatten())
model.add(keras.layers.Dense(12, activation=“relu“))
model.add(keras.layers.Dense(12, activation=“softmax“))
model.compile(optimizer=’rmsprop‘, loss=’categorical_crossentropy‘, metrics=[‚accuracy‘])
return model

def threshold(img):
img = img.copy()
img = img.astype(‚uint8‘)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
height , width = img.shape
img = cv2.bitwise_not(img)
blurSize = 2*int(height*0.01/2)+1
threshParameter = 2*int(height*0.9/2)+1
img = cv2.GaussianBlur(img, (blurSize, blurSize), 0)
img = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
cv2.THRESH_BINARY, threshParameter, -20)
return img

def findDigits(img):
original = img.copy()
height , width = img.shape[:2]
img = threshold(img)
_, contours, _ = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
splits = []
for x, y, w, h in box2rects(contours):
if h>height*0.05 and w>width*0.02:
split = original[max(0,y-10):y+h+20,max(0,x-10):x+w+20]
splits.append((split, (x, y, w, h)))
return splits

def getPrice(img):
splits = findDigits(img)
if len(splits)<2:
return 0.0
splits.sort(key= lambda split: split[1][0])
eur = []
cents = []
outputs = [recognizeDigit_digits(digit) for digit, _ in splits]
comma = [output[0][10] if np.argmax(output)==10 else -1 for output in outputs]
predictions = [ np.argmax(digit) for digit in outputs ]
if len(comma)>0:
i = np.argmax(comma)
eur = [str(pred) for pred in predictions[:i] if (pred!=11 and pred!=10)]
cents = [str(pred) for pred in predictions[i+1:] if (pred!=11 and pred!=10)]
try:
return float(„{}.{}“.format(„“.join(eur), „“.join(cents)))
except:
return 0.0
else:
eur = [str(pred) for pred in predictions if (pred!=11 and pred!=10)]
try:
return float(„“.join(eur))
except:
return 0.0

def recognizeDigit_digits(img):
img = img.copy()
img = img.astype(‚uint8‘)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
width, height = img.shape[:2]
img = addPadding(img, int(width*0.1), int(width*0.1), int(height*0.1), int(height*0.1))
img = pad_and_resize(img, 100, 50)
targetHeight = 100
targetWidth = 50
img = np.array(img).reshape([-1, targetHeight, targetWidth, 1])
img = np.array(img, dtype=np.float64)
img -= np.mean(img)
img /= np.std(img)
return model_digits.predict(img)

def show(img):
plt.imshow(img, cmap = ‚gray‘, interpolation = ‚bicubic‘)
plt.xticks([]), plt.yticks([]) # to hide tick values on X and Y axis
plt.show()

def result_model_digits():
directory = ‚data/cropped‘
with open(„results_model_digits_scores.csv“, ‚w‘) as w:
w.write(„Number;CNN_thresh;thresh_score\n“)
score_th = 0
score_n_th = 0
total = 0.0
suma = 0.0
nr = 0.0
for filename in os.listdir(directory):
total += 1
img = cv2.imread(os.path.join(directory, filename))
fname = filename.split(„.“)[0]
fname = fname.split(„_“)
fn = „{}.{}“.format(fname[0],fname[1])
start = time.time()
text_no_thresh = getPrice(img)
stop = time.time()
suma += stop-start
nr += 1

if float(fn) != float(text_no_thresh):
score_n_th += 1
no_thresh_score = 1
else :
no_thresh_score = 0
w.write(„{};{};{}\n“.format(fn,text_no_thresh,no_thresh_score))
print(„Average time {} sum {} number {}“.format(suma/nr,suma,nr))
w.write(„{};{};{}\n“.format(„“,“WRONG“,score_n_th))
w.write(„{};{};{}\n“.format(„“,“WRONG“,(score_n_th/total)*100))
w.write(„{};{};{}\n“.format(„“,“RIGHT“,100-score_n_th/total*100))

def threshold_receipt(img):
original = img.copy()
img = img.copy()
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
height , width = img.shape
img = cv2.bitwise_not(img)
blurSize = 2*int(width*0.01/2)+1
threshParameter = 2*int(width*0.7/2)+1
img = cv2.GaussianBlur(img, (blurSize, blurSize), 0)
img = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, threshParameter, -20)
return img

def findDigits_receipt(img):
original = img.copy()
height , width = img.shape[:2]
img = threshold_receipt(img)
_, contours, _ = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
splits = []
for x, y, w, h in box2rects(contours):
split = original[max(0,y-2):y+h+4,max(0,x-2):x+w+4]
splits.append((split, (x, y, w, h)))
return splits

def splitRows(img):
original = img.copy()
img = threshold_receipt(img)
height, width = img.shape
splits = []
for i in range(height):
if all([pixel==0 for pixel in img[i]]):
splits.append(i)
i = 0
rows = []
while i<height:
while i in splits:
i+=1
if i < height:
rows.append([])
while i not in splits and i < height:
rows[-1].append(original[i])
i+=1
rows = [np.array(row) for row in rows]
return rows

def recognizeDigit_receipt(img):
img = img.copy()
width, height = img.shape[:2]
img = pad_and_resize(img, 20, 10)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
targetHeight = 20
targetWidth = 10
img = np.array(img).reshape([-1, targetHeight, targetWidth, 1])
img = np.array(img, dtype=np.float64)
img -= np.mean(img)
img /= np.std(img)
return model_receipt.predict(img)

def getReceiptPrices(img):
rows = splitRows(img)
prices = []
for row in rows:
splits = findDigits_receipt(row)
if len(splits)<1:
continue
splits.sort(key= lambda split: split[1][0])
eur = []
cents = []
outputs = [recognizeDigit_receipt(digit) for digit, _ in splits]
comma = [output[0][10] if np.argmax(output)==10 else -1 for output in outputs]
predictions = [ np.argmax(digit) for digit in outputs ]
if len(comma)>0:
i = np.argmax(comma)
eur = [str(pred) for pred in predictions[:i] if pred!=11 and pred!=10]
cents = [str(pred) for pred in predictions[i+1:] if pred!=11 and pred!=10]
try:
prices.append(float(„{}.{}“.format(„“.join(eur), „“.join(cents))))
except Exception as e:
prices.append(0.0)
print(e)
else:
eur = [str(pred) for pred in predictions if pred!=11 and pred!=10]
try:
prices.append(float(„“.join(eur)))
except:
prices.append(0.0)
return prices

def receiptDigitRecognizer():
„““
Digit recognizer CNN model for the receipt digits
„““
model = keras.models.Sequential()
model.add(keras.layers.Conv2D(20, [2, 2], input_shape=[20, 10, 1], strides=(1, 1), padding=’valid‘, data_format=None, dilation_rate=(1, 1), activation=’relu‘, use_bias=True, kernel_initializer=’glorot_uniform‘, bias_initializer=’zeros‘, kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None))
model.add(keras.layers.MaxPooling2D(pool_size=(2, 2), strides=None, padding=’valid‘, data_format=None))
model.add(keras.layers.Dropout(0.5))
model.add(keras.layers.Flatten())
model.add(keras.layers.Dense(20, activation=“relu“))
model.add(keras.layers.Dense(12, activation=“softmax“))
model.compile(optimizer=’rmsprop‘, loss=’categorical_crossentropy‘, metrics=[‚accuracy‘])
return model

def result_model_receipt():
directory = ‚receiptData/photos‘
with open(„results_model_scores_receipt.csv“, ‚w‘) as w:
w.write(„Number;CNN\n“)
suma = 0.0
nr = 0.0
for filename in os.listdir(directory):
img = cv2.imread(os.path.join(directory, filename))
fname = filename.split(„.“)[0]
start = time.time()
text = getReceiptPrices(img)
stop = time.time()
suma += stop-start
nr += 1
w.write(„{};{}\n“.format(fname,text))
print(„Average time {} sum {} number {}“.format(suma/nr,suma,nr))

model_digits = digitRecognizer()
model_receipt = receiptDigitRecognizer()

def test_model_digits():
weightLocation = „digit_weights.h5“
model_digits.load_weights(weightLocation)
model_digits.summary()
start = time.time()
result_model_digits()
end = time.time()
print(end – start)

def test_model_receipt():
weightLocation = „receipt_weights.h5“
model_receipt.load_weights(weightLocation)
model_receipt.summary()
start = time.time()
result_model_receipt()
end = time.time()
print(end – start)

test_model_digits()
test_model_receipt()

Testing_tesseract.py

„““
Code created by Sorin Liviu Jurj and Allen-Jasmin Farcas for the paper called „Mobile Application for Receipt Fraud Detection Based on Optical Character Recognition“.
More information about the paper can be found here: https://www.jurj.de/mobile-application-for-receipt-fraud-detection-based-on-optical-character-recognition/
„““
import time
from PIL import Image
import pytesseract
import cv2
import os
import matplotlib.pyplot as plt

def threshold(img):
img = img.copy()
img = img.astype(‚uint8‘)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
height , width = img.shape
img = cv2.bitwise_not(img)
blurSize = 2*int(width*0.01/2)+1
threshParameter = 2*int(width*0.9/2)+1
img = cv2.GaussianBlur(img, (blurSize, blurSize), 0)
img = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, threshParameter, -20)
return img

def show(img):
plt.imshow(img, cmap = ‚gray‘, interpolation = ‚bicubic‘)
plt.xticks([]), plt.yticks([]) # to hide tick values on X and Y axis
plt.show()

def test_ocr_digits():
directory = ‚data/cropped‘
with open(„results_tesseract_scores_digits.csv“, ‚w‘) as w:
w.write(„Number;OCR_thresh;thresh_score;OCR_no_thresh;no_thresh_score\n“)
score_th = 0
score_n_th = 0
total = 0.0
suma = 0.0
nr = 0.0
for filename in os.listdir(directory):
total += 1
img = cv2.imread(os.path.join(directory, filename))
fname = filename.split(„.“)[0]
fname = fname.split(„_“)
fn = „{}“.format(fname[0])
start = time.time()
text_no_thresh = pytesseract.image_to_string(Image.fromarray(img))
stop = time.time()
suma += stop-start
nr += 1
if fn != text_no_thresh:
score_n_th += 1
no_thresh_score = 1
else :
no_thresh_score = 0
text_thresh = „“
thresh_score = „“
w.write(„{};{};{};{};{}\n“.format(fn,text_thresh,thresh_score,text_no_thresh,no_thresh_score))
print(„Average time {} sum {} number {}“.format(suma/nr,suma,nr))
w.write(„{};{};{};{};{}\n“.format(„“,“WRONG“,score_th,“WRONG“,score_n_th))
w.write(„{};{};{};{};{}\n“.format(„“,“WRONG“,(score_th/total)*100,“WRONG“,(score_n_th/total)*100))
w.write(„{};{};{};{};{}\n“.format(„“,“RIGHT“,100-score_th/total*100,“RIGHT“,100-score_n_th/total*100))

def test_ocr_receipt():
directory = ‚receiptData/photos‘
with open(„results_tesseract_scores_receipt.csv“, ‚w‘) as w:
w.write(„Number;OCR_thresh;OCR_no_thresh\n“)
suma = 0.0
nr = 0.0
for filename in os.listdir(directory):
img = cv2.imread(os.path.join(directory, filename))
fname = filename.split(„.“)[0]
fname = fname.split(„_“)
fn = „{}“.format(fname[0])
start = time.time()
text_no_thresh = pytesseract.image_to_string(Image.fromarray(img))
stop = time.time()
suma += stop-start
nr += 1
text_no_thresh = text_no_thresh.split(„\n“)
text_thresh = „“
w.write(„{};{};{}\n“.format(fname,text_thresh,text_no_thresh))
print(„Average time {} sum {} number {}“.format(suma/nr,suma,nr))

def result_ocr():
start = time.time()
test_ocr_digits()
end = time.time()
print(end – start)

start = time.time()
test_ocr_receipt()
end = time.time()
print(end – start)

result_ocr()

KivyApplication.py

„““
Code created by Sorin Liviu Jurj for the paper called „Mobile Application for Receipt Fraud Detection Based on Optical Character Recognition“.
More information about the paper can be found here: https://www.jurj.de/mobile-application-for-receipt-fraud-detection-based-on-optical-character-recognition/
„““
from kivy.lang import Builder
from kivy.app import App
from kivy.uix.widget import Widget
from kivy.graphics import Rectangle, Color, Point
from kivy.uix.dropdown import DropDown
from kivy.properties import NumericProperty, StringProperty, ListProperty, ObjectProperty
from kivy.uix.screenmanager import ScreenManager, Screen
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.behaviors import ButtonBehavior
from kivy.uix.image import Image
from kivy.uix.camera import Camera
from kivy.uix.gridlayout import GridLayout
from kivy.uix.scrollview import ScrollView
from kivy.core.window import Window
from kivy.uix.modalview import ModalView
import kivy.uix.boxlayout
import kivy.uix.textinput
import kivy.uix.label
import kivy.uix.button
import cv4
import numpy as np
import os
import pickle

# Button and highlight colors
GREEN_COLOR = [0, 0.5, 0, 1]
RED_COLOR = [0.5, 0, 0, 1]
ADD_ITEM_COLOR = [0, 1, 0, 1]
FINISH_COLOR = [1, 0, 0, 1]
RECEIPT_BUTTONS_COLOR = [0.8, 0.8, 1, 1]

# Background color
Window.clearcolor = (1, 1, 1, 1)

# Photo location
app_folder = os.path.realpath(‚.‘)

Builder.load_string(“‘
<Button>:
background_color: [0.5, 0.9, 0.9, 1] #ALL THE OTHER BUTTONS
<CameraClick>:
orientation: ‚vertical‘
Camera:
id: camera
resolution: (640, 480)
play: True
canvas.before:
PushMatrix
Rotate:
angle: -90
origin: self.center
canvas.after:
PopMatrix
<ImageContainer>:
ImageButton:
id: image
source: „NoPicture.tmp“
center_x: self.parent.center_x
center_y: self.parent.center_y
size: self.parent.height, self.parent.width
on_press: self.parent.press(self)
<EditableLabel>:
on_press: self.change()
<Label>:
color: 0, 0, 0, 1
<CroppingScreen>:
orientation: „vertical“
BoxLayout:
id: button_boxlayout
orientation: „horizontal“
padding: 0
size_hint: (1, 0.15)
Button:
id: accept_button
text: „Crop“
on_press: root.finishCropping(self)
ThirdScreen:
name: ‚_third_screen_‘
id: third_screen
BoxLayout:
orientation: „vertical“
id: third_screen_boxlayout
BoxLayout:
id: image_box_layout
size: self.parent.size
Image:
id: main_image
source: root.source
Widget:
id: image_canvas
size_hint: (0.0000001, 0.0000001)
canvas:
Color:
rgba: (1, 1, 1)
Rectangle:
id: root.rect_box
pos: (root.x1-2, root.y1-2)
size: (root.t_x+4, 5)
Rectangle:
id: root.rect_box
pos: (root.x1-2, root.y1-2)
size: (5, root.t_y+4)
Rectangle:
id: root.rect_box
pos: (root.x1-2, root.y1+root.t_y-2)
size: (root.t_x+4, 5)
Rectangle:
id: root.rect_box
pos: (root.x1+root.t_x-2, root.y1-2)
size: (5, root.t_y-4)
Color:
rgb: (0,0,0)
Rectangle:
id: root.rect_box
pos: (root.x1-1, root.y1-1)
size: (root.t_x+2, 3)
Rectangle:
id: root.rect_box
pos: (root.x1-1, root.y1-1)
size: (3, root.t_y+2)
Rectangle:
id: root.rect_box
pos: (root.x1-1, root.y1+root.t_y-1)
size: (root.t_x+2, 3)
Rectangle:
id: root.rect_box
pos: (root.x1+root.t_x-1, root.y1-1)
size: (3, root.t_y+2)

“‘)

class CropView(ModalView):
# Add in the CropView a CroppingScreen
def __init__(self, source, on_close):
super().__init__()
self.add_widget(CroppingScreen(source, on_close))

class CroppingScreen(BoxLayout):
source = StringProperty(„“)
t_x = NumericProperty(0.0)
t_y = NumericProperty(0.0)
x1 , y1 , x2 , y2 = NumericProperty(0.0), NumericProperty(0.0), NumericProperty(0.0), NumericProperty(0.0)

def __init__(self, source, on_close):
super().__init__()
self.source = source
self.on_close = on_close

# select the cropped image
def finishCropping(self, btn):
btn.parent.remove_widget(btn)
img = cv4.imread(os.path.join(app_folder, self.source))
beginX, endX = int(min(self.x1, self.x2)*img.shape[1]/self.ids[„third_screen“].size[0]), int(max(self.x1, self.x2)*img.shape[1]/self.ids[„third_screen“].size[0])
beginY, endY = int(min(self.y1, self.y2)*img.shape[0]/self.ids[„third_screen“].size[1]), int(max(self.y1, self.y2)*img.shape[0]/self.ids[„third_screen“].size[1])
img = img[-endY:-beginY, beginX:endX]
self.on_close(img)
self.parent.dismiss()

# The rectangle used for cutting the photo
class ThirdScreen(Screen):
rect_box = ObjectProperty(None)
t_x = NumericProperty(0.0)
t_y = NumericProperty(0.0)
x1 , y1 , x2 , y2 = NumericProperty(0.0), NumericProperty(0.0), NumericProperty(0.0), NumericProperty(0.0)

# Moving the rectangle’s edges
def on_touch_down(self, touch):
if self.collide_point(*touch.pos):
touch.grab(self)
self.parent.x1 = touch.x
self.parent.y1 = touch.y
self.parent.t_x = 0
self.parent.t_y = 0

def on_touch_move(self, touch):
if touch.grab_current is self:
self.parent.t_x = touch.x-self.parent.x1
self.parent.t_y = touch.y-self.parent.y1

def on_touch_up(self, touch):
if touch.grab_current is self:
self.parent.x2 = touch.x
self.parent.y2 = touch.y

class ImageButton(ButtonBehavior, Image):
pass

class CameraClick(BoxLayout):
def capture(self, path):
“‘
Function to capture the images and give them the names
according to their captured time and date.
“‘
camera = self.ids[‚camera‘]
camera.export_to_png(os.path.join(app_folder,path))

# Camera Object for taking photos
CAM = CameraClick(size_hint_y=9)

def box2rects(contours):
„““
Return the bounding rectangles of the input countours
„““
return [cv4.boundingRect(c) for c in contours]

def npload(file):
„““
Function for loading the pickle serialized files
„““
with open(file, ‚rb‘) as f:
return pickle.load(f)

def compareList(rows, valueFunction, elements, trueFunction, falseFunction):
„““
Function for comparing the products with the receipt
„““
failedRowsValues = []
print([valueFunction(row) for row in rows], elements)
for row in rows:
if valueFunction(row) in elements:
trueFunction(row)
elements.remove(valueFunction(row))
else:
falseFunction(row)
failedRowsValues.append(valueFunction(row))

for row in rows:
if valueFunction(row) in failedRowsValues:
falseFunction(row)

# Class for all CNN functions
class DigitRecognizer:
def __init__(self, number):
self.conv2dWeights = np.transpose(np.array(npload(f“CNN{number}_conv2d_weights.pickle“)))
self.conv2dBias = np.array(npload(f“CNN{number}_conv2d_bias.pickle“))
self.dense1Weights = np.array(npload(f“CNN{number}_dense1_weights.pickle“))
self.dense1Bias = np.array(npload(f“CNN{number}_dense1_bias.pickle“))
self.dense2Weights = np.array(npload(f“CNN{number}_dense2_weights.pickle“))
self.dense2Bias = np.array(npload(f“CNN{number}_dense2_bias.pickle“))
self.version = number
if number == 1:
self.strides = 2
else:
self.strides = 1

def applyCNN(self, image):
„““
Inference function
„““
conv = relu(conv2d(image, self.conv2dWeights, self.conv2dBias, [self.strides, self.strides]))
mpool = max_pooling( conv , [2,2])
flat = flatten(mpool)
dense = relu(dense_forward(flat, self.dense1Weights, self.dense1Bias))
return softmax(dense_forward(dense ,self.dense2Weights, self.dense2Bias))

def recognizeDigit(self, img):
„““
Digit preparation and recognition function
„““
img = img.copy()
width, height = img.shape[:2]
if self.version == 1:
img = addPadding(img, int(width*0.1), int(width*0.1), int(height*0.1), int(height*0.1))
img = pad_and_resize(img, 100, 50)
else:
img = pad_and_resize(img, 20, 10)
img = cv4.cvtColor(img, cv4.COLOR_BGR2GRAY)
if self.version ==1:
targetHeight = 100
targetWidth = 50
else:
targetHeight = 20
targetWidth = 10
img = np.array(img).reshape([targetHeight, targetWidth, 1])
return self.applyCNN(img)

def threshold(self, img):
„““
Image preprocessing before digit extraction function
„““
original = img.copy()
img = img.copy()
img = cv4.cvtColor(img, cv4.COLOR_BGR2GRAY)
height , width = img.shape
img = cv4.bitwise_not(img)
if self.version==1:
blurSize = 2*int(height*0.01/2)+1
threshParameter = 2*int(height*0.9/2)+1
else:
blurSize = 2*int(width*0.01/2)+1
threshParameter = 2*int(width*0.7/2)+1
img = cv4.GaussianBlur(img, (blurSize, blurSize), 0)
img = cv4.adaptiveThreshold(img, 255, cv4.ADAPTIVE_THRESH_GAUSSIAN_C, cv4.THRESH_BINARY, threshParameter, -20)
return img

def findDigits(self, img):
„““
Digit extraction from images function
„““
original = img.copy()
height , width = img.shape[:2]
img = self.threshold(img)
contours, _ = cv4.findContours(img, cv4.RETR_EXTERNAL, cv4.CHAIN_APPROX_NONE)
splits = []
for x, y, w, h in box2rects(contours):
if self.version ==1:
if h>height*0.05 and w>width*0.02:
split = original[max(0,y-10):y+h+20,max(0,x-10):x+w+20]
splits.append((split, (x, y, w, h)))
else:
split = original[max(0,y-2):y+h+4,max(0,x-2):x+w+4]
splits.append((split, (x, y, w, h)))
return splits

def splitRows(self, img):
„““
Receipt image row split function
„““
original = img.copy()
img = self.threshold(img)
height, width = img.shape
splits = []
for i in range(height):
if all([pixel==0 for pixel in img[i]]):
splits.append(i)
i = 0
rows = []
while i<height:
while i in splits:
i+=1
if i < height:
rows.append([])
while i not in splits and i < height:
rows[-1].append(original[i])
i+=1
rows = [np.array(row) for row in rows]
return rows

def getReceiptPrices(self, img):
„““
Receipt price recognition function
„““
rows = self.splitRows(img)
prices = []
for row in rows:
splits = self.findDigits(row)
if len(splits)<1:
continue
splits.sort(key= lambda split: split[1][0])
eur = []
cents = []
outputs = [self.recognizeDigit(digit) for digit, _ in splits]
comma = [output[10] if np.argmax(output)==10 else -1 for output in outputs]
predictions = [ np.argmax(digit) for digit in outputs ]
if len(comma)>0:
i = np.argmax(comma)
eur = [str(pred) for pred in predictions[:i] if pred!=11 and pred!=10]
cents = [str(pred) for pred in predictions[i+1:] if pred!=11 and pred!=10]
try:
prices.append(float(„{}.{}“.format(„“.join(eur), „“.join(cents))))
except Exception as e:
prices.append(0.0)
print(e)
else:
eur = [str(pred) for pred in predictions if pred!=11 and pred!=10]
try:
prices.append(float(„“.join(eur)))
except:
prices.append(0.0)
return prices

def getPrice(self, img):
„““
Product price recognition function
„““
splits = self.findDigits(img)
if len(splits)<2:
return 0.0
splits.sort(key= lambda split: split[1][0])
eur = []
cents = []
outputs = [self.recognizeDigit(digit) for digit, _ in splits]
comma = [output[10] if np.argmax(output)==10 else -1 for output in outputs]
predictions = [ np.argmax(digit) for digit in outputs ]
if len(comma)>0:
i = np.argmax(comma)
eur = [str(pred) for pred in predictions[:i] if (pred!=11 and pred!=10)]
cents = [str(pred) for pred in predictions[i+1:] if (pred!=11 and pred!=10)]
try:
return float(„{}.{}“.format(„“.join(eur), „“.join(cents)))
except:
return 0.0
else:
eur = [str(pred) for pred in predictions if (pred!=11 and pred!=10)]
try:
return float(„“.join(eur))
except:
return 0.0

# The two CNN’s objects
priceTagRecognizer = DigitRecognizer(1)
receiptRecognizer = DigitRecognizer(2)

def addPadding(img, pad1, pad2, pad3, pad4):
„““
Add a padding to the input image according to the specified amounts
„““
return cv4.copyMakeBorder(img, pad3, pad4, pad1, pad2, cv4.BORDER_REPLICATE)

def pad_and_resize(img, height, width):
„““
Rezises and pad the image in order to return an image of the target height and width
„““
img = img.copy()
desiredRatio = height/width
ratio = img.shape[0]/img.shape[1]
if ratio == desiredRatio:
return cv4.resize(img, dsize=(width, height))
elif ratio > desiredRatio:
img = cv4.resize(img, dsize=(max(1, int(height*img.shape[1]/img.shape[0])), height))
delta_w = width – img.shape[1]
img = cv4.copyMakeBorder(img, 0, 0, delta_w//2, delta_w-delta_w//2, cv4.BORDER_CONSTANT, value=[255, 255, 255])
assert img.shape[0] == height
assert img.shape[1] == width
return img
else:
img = cv4.resize(img, dsize=(width, max(1,int(width*img.shape[0]/img.shape[1]))))
delta_h = height – img.shape[0]
img = cv4.copyMakeBorder(img, delta_h//2, delta_h-delta_h//2, 0, 0, cv4.BORDER_CONSTANT, value=[255, 255, 255])
assert img.shape[0] == height
assert img.shape[1] == width
return img

def conv2d(inp, filters, bias, strides):
„““
Convolutional layer function
„““
inp = np.transpose(inp)
assert len(inp.shape)==3
assert len(filters.shape)==4
assert filters.shape[1] == inp.shape[0]
assert len(strides)==2
assert len(bias.shape)==1
assert bias.shape[0] == filters.shape[0]
s1, s2 = strides
f1, f2 = filters.shape[2:]
outputShape = [filters.shape[0], int((inp.shape[1]-filters.shape[2])/strides[0]) + 1,
int((inp.shape[2]-filters.shape[3])/strides[1]) + 1]
output = np.zeros(outputShape)
for d in range(outputShape[0]):
for i in range(outputShape[1]):
for j in range(outputShape[2]):
output[d, i, j] = bias[d] +
np.sum(np.multiply(filters[d], inp[:, i*s1:i*s1 + f1, j*s2:j*s2+f2 ]) )
return output

def max_pooling(inp, pool_size):
„““
Max Pooling layer function
„““
assert len(inp.shape)==3
assert len(pool_size)==2
w=pool_size[0]
h=pool_size[1]
outputShape = list(inp.shape)
outputShape[1] = int(outputShape[1]/pool_size[0])
outputShape[2] = int(outputShape[2]/pool_size[1])
output = np.zeros(outputShape)
for d in range(outputShape[0]):
for i in range(outputShape[1]):
for j in range(outputShape[2]):
output[d,i,j] = np.max(inp[d,w*i:w*i+w, h*j:h*j+h])

return output

def flatten(img):
„““
Flattening function
„““
return img.flatten(‚F‘)

def dense_forward(inp, weights, bias):
„““
FC layer function
„““
assert len(inp.shape)==1
assert len(bias.shape)==1
assert len(weights.shape)==2
assert inp.shape[0] == weights.shape[0]
assert weights.shape[1] == bias.shape[0]
return np.matmul(inp, weights)+bias

def relu(a):
„““
ReLU activation function
„““
return a * (a>0)

def softmax(x):
„““
Softmax activation function
„““
return np.exp(x)/np.sum(np.exp(x))

# Receipt related processing
class ReceiptView(kivy.uix.boxlayout.BoxLayout):
def __init__(self):
super().__init__(orientation = „vertical“)
self.bar = kivy.uix.boxlayout.BoxLayout(orientation=“horizontal“)
self.addReceiptButton = kivy.uix.button.Button(text=“Add receipt“, background_color=RECEIPT_BUTTONS_COLOR)
self.addReceiptButton.bind(on_press = self.addReceipt)
self.bar.add_widget(self.addReceiptButton)
self.cropButton = kivy.uix.button.Button(text=“Crop receipt“, background_color=RECEIPT_BUTTONS_COLOR)
self.bar.add_widget(self.cropButton)
self.bar.add_widget(kivy.uix.label.Label(text=“Prices“, size_hint_x=0.5))
self.add_widget(self.bar)
self.body = kivy.uix.boxlayout.BoxLayout(orientation=“horizontal“, size_hint_y=8)
self.add_widget(self.body)
self.returnButton = kivy.uix.button.Button(text=“Return to items“, background_color=FINISH_COLOR)
self.returnButton.bind(on_press=self.finish)
self.add_widget(self.returnButton)
self.detectedPrices = Widget()

def finish(self, btn):
„““
When Return to Items button tapped compare results
„““
btn.parent.parent.compareResults()
btn.parent.parent.changeView()

def addReceipt(self, btn):
„““
Add receipt by taking a photo of it
„““
def on_changed():
self.receiptImageButton.ids[„image“].source = „“
self.receiptImageButton.ids[„image“].source = os.path.join(app_folder, „receiptImage.png“)
self.receiptImageButton.ids[„image“].reload()
def on_close():
def newPicture(btn):
def on_change():
cam = CameraWidget(„receiptImage.png“, on_changed)
cam.open()
view = PictureViewer(os.path.join(app_folder, „receiptImage.png“), on_change)
view.open()
self.receiptImageButton = ImageContainer(os.path.join(app_folder, „receiptImage.png“), newPicture)
self.body.add_widget(self.receiptImageButton)
self.addReceipt = lambda x: x
self.cropButton.bind(on_press=self.cropReceipt)
cam = CameraWidget(„receiptImage.png“, on_close)
cam.open()

def cropReceipt(self, btn):
„““
Crop the photographed receipt
„““
def on_changed(img):
cv4.imwrite(„croppedReceipt.png“, img)
prices = receiptRecognizer.getReceiptPrices(img)
self.detectedPrices.clear_widgets()
for price in prices:
self.detectedPrices.add_widget(EditableLabel(text=str(price)))
self.viewCroppedReceipt.ids[„image“].source = „“
self.viewCroppedReceipt.ids[„image“].source = os.path.join(app_folder, „croppedReceipt.png“)
self.viewCroppedReceipt.ids[„image“].reload()
def on_close(img):
self.cropReceipt = lambda x: x
cv4.imwrite(„croppedReceipt.png“, img)
def newPicture(btn):
def on_change():
crop = CropView(„receiptImage.png“, on_changed)
crop.open()
view = PictureViewer(os.path.join(app_folder, „croppedReceipt.png“), on_change)
view.open()
prices = receiptRecognizer.getReceiptPrices(img)
self.viewCroppedReceipt = ImageContainer(os.path.join(app_folder, „croppedReceipt.png“), newPicture)
self.detectedPrices = kivy.uix.boxlayout.BoxLayout(orientation=“vertical“, size_hint_x=0.5)
for price in prices:
self.detectedPrices.add_widget(EditableLabel(text=str(price)))
self.body.add_widget(self.viewCroppedReceipt)
self.body.add_widget(self.detectedPrices)
crop = CropView(„receiptImage.png“, on_close)
crop.open()

# Main products view
class ItemSelectionView(kivy.uix.boxlayout.BoxLayout):
def __init__(self):
super().__init__(orientation=“vertical“)
self.statusBar = StatusBar()
self.add_widget(self.statusBar)
rowLayout = kivy.uix.boxlayout.BoxLayout(orientation=“horizontal“, size_hint=[1,0.5])
pictures = kivy.uix.label.Label(text=“Pictures“)
detectedPrice = kivy.uix.label.Label(text=“Unit price“)
amount = kivy.uix.label.Label(text=“Amount“, size_hint_x=0.25)
totalPrice = kivy.uix.label.Label(text=“Total price“)
rowLayout.add_widget(pictures)
rowLayout.add_widget(detectedPrice)
rowLayout.add_widget(amount)
rowLayout.add_widget(totalPrice)
self.add_widget(rowLayout)
self.scroll = ScrollView(size_hint=(1, None), size=(Window.width, int(Window.height*0.75)))
self.layout = kivy.uix.boxlayout.BoxLayout(orientation=“vertical“, size_hint_y=None, spacing=1)
self.layout.bind(minimum_height=self.layout.setter(‚height‘))
self.scroll.add_widget(self.layout)
self.add_widget(self.scroll)
self.finishBtn = kivy.uix.button.Button(text=“Finish“, background_color=(1,0,0,1))
self.finishBtn.bind(on_press=self.finish)
self.add_widget(self.finishBtn)
self.numberOfItems = 0

def finish(self, btn):
„““
Change to ReceiptView
„““
btn.parent.parent.changeView()

def addItem(self):
„““
Add products
„““
self.numberOfItems += 1
self.layout.add_widget(ItemRow(str(self.numberOfItems)))

def updateTotal(self):
„““
Update cart total
„““
total = 0
for child in self.layout.children:
total += child.totalPrice
self.statusBar.totalPrice = total
self.statusBar.totalPriceLabel.text = “ Total price = {}“.format(str(total))

# Titles and cart total price
class StatusBar(kivy.uix.boxlayout.BoxLayout):
totalPrice = NumericProperty(0)
def __init__(self):
super().__init__(orientation=“horizontal“, size_hint=[1,None], size=[Window.width, Window.height*0.1])
self.btn = kivy.uix.button.Button(text=“Add item“, background_color=ADD_ITEM_COLOR)
self.btn.bind(on_press=self.action)
self.add_widget(self.btn)
self.totalPriceLabel = kivy.uix.label.Label(text=“ Total price = {}“.format(str(self.totalPrice)))
self.add_widget(self.totalPriceLabel)

def action(self, non):
self.parent.addItem()

# Class for each element from ItemSelectionView list
class ItemRow(kivy.uix.boxlayout.BoxLayout):
detectedPrice = NumericProperty(0)
amount = NumericProperty(1)
totalPrice = NumericProperty(0)
def __init__(self, name):
super().__init__(orientation=“horizontal“, size_hint=[1,None])
self.pictures = PictureButtons(name)
self.detectedPriceLabel = EditableLabel()
self.dropdown = DropDown()
self.amountInput = kivy.uix.button.Button(text=’X1′, size_hint_x=0.5)
self.amountInput.bind(on_release=self.dropdown.open)
def ddownOnSel(instance, x):
setattr(self.amountInput, ‚text‘, x)
self.amount = int(x[1:])
if self.detectedPrice > 0:
self.totalPrice = self.detectedPrice*self.amount
self.totalPriceLabel.text = str(self.totalPrice)
self.parent.parent.parent.updateTotal()
self.dropdown.bind(on_select=ddownOnSel)
for index in range(1,100):
btn = kivy.uix.button.Button(text=’X{}‘.format(index), size_hint_y=None)
btn.bind(on_release=lambda btn: self.dropdown.select(btn.text))
self.dropdown.add_widget(btn)
self.totalPriceLabel = kivy.uix.label.Label(text=““)
self.add_widget(self.pictures)
self.add_widget(self.detectedPriceLabel)
self.add_widget(self.amountInput)
self.add_widget(self.totalPriceLabel)
with self.canvas.before:
self.color_widget = Color(0.9, 0.9, 0.9, 1)
self._rectangle = Rectangle(size=self.size, pos=self.pos)
self.bind(pos=self.update_rect, size=self.update_rect)
self.height = int(Window.height*0.2)

def update_rect(self, instance, value):
instance._rectangle.pos = instance.pos
instance._rectangle.size = instance.size

def changeColor(self, r, g, b, a):
with self.canvas.before:
self.color_widget = Color(r, g, b, a)
self._rectangle = Rectangle(size=self.size, pos=self.pos)

def on_size(self, *args):
self._rectangle.size = self.size
self._rectangle.pos = self.pos

# Make the pictures to react to touch and open for editing
class PictureButtons(kivy.uix.boxlayout.BoxLayout):
def __init__(self, name):
super().__init__(orientation=“vertical“)
self.name = name
self.addItemPictureButton = kivy.uix.button.Button(text=“Add picture“)
self.addItemPictureButton.bind(on_press=self.addItemPicture)
self.addItemPriceTagPictureButton = kivy.uix.button.Button(text=“Add tag“)
self.addItemPriceTagPictureButton.bind(on_press=self.addItemPriceTagPicture)
self.add_widget(self.addItemPictureButton)
self.add_widget(self.addItemPriceTagPictureButton)
self.cropPriceTagButton = kivy.uix.button.Button(text=“Crop price“)
self.cropPriceTagButton.bind(on_press=self.cropPriceTag)

def addItemPicture(self, btn):
„““
Add picture of product
„““
def on_changed():
self.viewItemPictureButton.ids[„image“].source = „“
self.viewItemPictureButton.ids[„image“].source = os.path.join(app_folder,f“itemPicture_{self.name}.png“)
self.viewItemPictureButton.ids[„image“].reload()
def on_close():
self.remove_widget(btn)
def newPicture(btn):
def on_change():
cam = CameraWidget(f“itemPicture_{self.name}.png“, on_changed)
cam.open()
view = PictureViewer(os.path.join(app_folder,f“itemPicture_{self.name}.png“), on_change)
view.open()
self.viewItemPictureButton = ImageContainer(os.path.join(app_folder,f“itemPicture_{self.name}.png“), newPicture)
self.add_widget(self.viewItemPictureButton, index=1)
cam = CameraWidget(f“itemPicture_{self.name}.png“, on_close)
cam.open()

def addItemPriceTagPicture(self, btn):
„““
Add picture of product price tag
„““
def on_changed():
self.viewPriceTagPictureButton.ids[„image“].source = „“
self.viewPriceTagPictureButton.ids[„image“].source = os.path.join(app_folder, f“priceTagPicture_{self.name}.png“)
self.viewPriceTagPictureButton.ids[„image“].reload()
def on_close():
self.remove_widget(btn)
def newPicture(btn):
def on_change():
cam = CameraWidget(f“priceTagPicture_{self.name}.png“, on_changed)
cam.open()
view = PictureViewer(os.path.join(app_folder, f“priceTagPicture_{self.name}.png“), on_change)
view.open()
self.viewPriceTagPictureButton = ImageContainer(os.path.join(app_folder, f“priceTagPicture_{self.name}.png“), newPicture)
self.add_widget(self.viewPriceTagPictureButton, index=2)
self.add_widget(self.cropPriceTagButton)
cam = CameraWidget(f“priceTagPicture_{self.name}.png“, on_close)
cam.open()

def cropPriceTag(self, btn):
„““
Crop the product price tag
„““
def on_changed(img):
cv4.imwrite(f“croppedTagPicture_{self.name}.png“, img)
self.viewCroppedPriceTagPictureButton.ids[„image“].source = „“
self.viewCroppedPriceTagPictureButton.ids[„image“].source = os.path.join(app_folder, f“croppedTagPicture_{self.name}.png“)
self.viewCroppedPriceTagPictureButton.ids[„image“].reload()
price = priceTagRecognizer.getPrice(img)
self.parent.detectedPrice = price
self.parent.detectedPriceLabel.text = str(price)
self.parent.totalPrice = price*self.parent.amount
self.parent.totalPriceLabel.text = str(self.parent.totalPrice)
self.parent.parent.parent.parent.updateTotal()
print(self.parent, self.parent.detectedPrice)
def on_close(img):
cv4.imwrite(f“croppedTagPicture_{self.name}.png“, img)
self.remove_widget(btn)
def newPicture(btn):
def on_change():
crop = CropView(f“priceTagPicture_{self.name}.png“, on_changed)
crop.open()
view = PictureViewer(os.path.join(app_folder, f“croppedTagPicture_{self.name}.png“), on_change)
view.open()
price = priceTagRecognizer.getPrice(img)
self.parent.detectedPrice = price
self.parent.detectedPriceLabel.text = str(price)
self.parent.totalPrice = price*self.parent.amount
self.parent.totalPriceLabel.text = str(price*self.parent.amount)
self.parent.parent.parent.parent.updateTotal()
print(self.parent, self.parent.detectedPrice)
self.viewCroppedPriceTagPictureButton = ImageContainer(os.path.join(app_folder, f“croppedTagPicture_{self.name}.png“), newPicture)
self.add_widget(self.viewCroppedPriceTagPictureButton)
crop = CropView(f“priceTagPicture_{self.name}.png“, on_close)
crop.open()

# Class for images to be hold into
class ImageContainer(kivy.uix.boxlayout.BoxLayout):
press = ObjectProperty(lambda x: print(x) )
def __init__(self, source, press):
super().__init__()
self.press = press
self.ids[„image“].source = source
def changeSource(self, source):
self.ids[„image“].source = source

# Camera widget insertion
class CameraWidget(ModalView):
def __init__(self, path, on_close):
super().__init__()
self.path = path
self.box = BoxLayout(orientation=“vertical“)
self.button = kivy.uix.button.Button(text=“Take picture“)
self.button.bind(on_press=self.take_picture)
self.add_widget(self.box)
self.box.add_widget(CAM)
self.box.add_widget(self.button)
self.on_close = on_close

def take_picture(self, btn):
„““
Take a photo with the phone’s camera
„““
CAM.capture(self.path)
self.on_close()
self.box.remove_widget(CAM)
self.dismiss()

# View the picture and change it
class PictureViewer(ModalView):
def __init__(self, source, on_change):
super().__init__()
self.box = BoxLayout(orientation=“vertical“)
self.button = kivy.uix.button.Button(text=“Cancel“)
self.button.bind(on_press=self.close)
self.buttons = BoxLayout(orientation=“horizontal“)
self.add_widget(self.box)
self.buttons.add_widget(self.button)
self.changeButton = kivy.uix.button.Button(text=“Edit“)
self.changeButton.bind(on_press=self.change)
self.buttons.add_widget(self.changeButton)
self.box.add_widget(self.buttons)
self.image = Image(source=source, size_hint_y=9)
self.box.add_widget(self.image)
self.on_change = on_change

def close(self, btn):
self.dismiss()

def change(self, btn):
self.on_change()
self.dismiss()

# Editable labels class for correcting labels with errors detected
class EditableLabel(ButtonBehavior, kivy.uix.boxlayout.BoxLayout):
text = StringProperty(„“)
def __init__(self, **kwargs):
super().__init__(**kwargs, orientation=“vertical“)
self.input = kivy.uix.textinput.TextInput(text = self.text)
self.label = kivy.uix.label.Label(text = self.text)
self.add_widget(self.label)
self.bind(text = self.label.setter(‚text‘))

def change(self):
if self.input in self.children:
self.remove_widget(self.input)
self.text = self.input.text
self.label.text = self.text
if isinstance(self.parent, ItemRow) and float(self.text) > 0:
self.parent.detectedPrice = float(self.text)
self.parent.totalPrice = self.parent.detectedPrice*self.parent.amount
self.parent.totalPriceLabel.text = str(self.parent.totalPrice)
self.parent.parent.parent.parent.updateTotal()
else:
self.text = self.label.text
self.input.text = self.text
self.add_widget(self.input)

# Main class with the two main views ReceiptView and ItemSelectionView
class MainWidget(kivy.uix.boxlayout.BoxLayout):
receiptView = ReceiptView()
itemView = ItemSelectionView()
def __init__(self):
super().__init__()
self.add_widget(self.itemView)

def changeView(self):
„““
Changing between the views
„““
if self.receiptView in self.children:
self.remove_widget(self.receiptView)
self.add_widget(self.itemView)
else:
self.remove_widget(self.itemView)
self.add_widget(self.receiptView)

def compareResults(self):
„““
Price comparing function
„““
rows = self.itemView.layout.children
def rowValue(row):
return row.totalPrice
def colorGreen(row):
row.changeColor(*GREEN_COLOR)
def colorRed(row):
row.changeColor(*RED_COLOR)
receiptValues = [float(priceLabel.text) for priceLabel in self.receiptView.detectedPrices.children]
compareList(rows, rowValue, receiptValues, colorGreen, colorRed)

# Main class for the entire application
class MainApp(App):
def build(self):
return MainWidget()

# Run the application
if __name__ == „__main__“:
MainApp().run()

Leave a Comment

Diese Website verwendet Akismet, um Spam zu reduzieren. Erfahren Sie mehr darüber, wie Ihre Kommentardaten verarbeitet werden .