Imports & dependencies
In [54]:
import tensorflow as tf
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score, classification_report
from helper_functions import create_tensorboard_callback, plot_loss_curves, unzip_data, compare_historys, walk_through_dir
import pandas as pd
import numpy as np
import os
import randomDownload Pre-Trained Model
In [6]:
# Download a pre-trained model from the web via Google Storage
!wget https://storage.googleapis.com/ztm_tf_course/food_vision/06_101_food_class_10_percent_saved_big_dog_model.zip
saved_model_path = "06_101_food_class_10_percent_saved_big_dog_model.zip"
unzip_data(saved_model_path)In [7]:
# Note: loading a model will output a lot of 'WARNINGS', these can be ignored:
# https://www.tensorflow.org/tutorials/keras/save_and_load#save_checkpoints_during_training
# There's also a thread on GitHub trying to fix these warnings: https://github.com/tensorflow/tensorflow/issues/40166
m0 = tf.keras.models.load_model(saved_model_path.split(".")[0]) # don't include ".zip" in loaded model pathPrepare Testing data
In [8]:
imagesDirPath = "101_food_classes_10_percent/"
testDirPath = imagesDirPath + 'test/'
print(f'TEST dir: {testDirPath}')In [9]:
OUTPUT_IMG_SIZE = (224,224)
testingData10p = tf.keras.preprocessing.image_dataset_from_directory(testDirPath,
label_mode="categorical",
image_size=OUTPUT_IMG_SIZE,
shuffle=False) # don't shuffle test data for prediction analysisEvaluate Model on Testing Data
In [10]:
# Check to see if loaded model is a trained model
loaded_loss, loaded_accuracy = m0.evaluate(testingData10p)
loaded_loss, loaded_accuracyOut [10]:
Make Predictions On New Images
The predict() method can be used to make predictions with the model on new data.
The data will have 101 classes.
Each prediction will return a prediction probability tensor for each class.
In [11]:
# Make predictions with model
pred_probs = m0.predict(testingData10p, verbose=1) # set verbosity to see how long it will take Inspect The Prediction results
In [12]:
# ensure the number of predictions matches the above prediction dataset length
len(pred_probs)Out [12]:
In [13]:
# What's the shape of our predictions?
# should return (numberOfImages, numberOfClasses)
pred_probs.shapeOut [13]:
In [14]:
# How do they look?
pred_probs[:5]Out [14]:
In [16]:
imageIndex = 0
aPrediction = pred_probs[imageIndex]
# We get one prediction probability per class
print(f"Number of prediction probabilities for sample {imageIndex}: {len(aPrediction)}")
print(f"What prediction probability sample {imageIndex} looks like:\n {aPrediction}")
print(f"The class with the highest predicted probability by the model for sample {imageIndex}: {aPrediction.argmax()}")Each prediction value is a number between 0 and 1.
The highest prediction value out of the 101 values (classes) is the "winner" that the model chooses as the most likely class match.
Get & Inspect Classes
In [17]:
# Get the class predicitons of each label
pred_classes = pred_probs.argmax(axis=1)
# How do they look?
pred_classes[:5]Out [17]:
Compare Predictions To The Real Results
- we have a prediction set
- we have classes
- we have the testing data
Now:
- "unwrap" the test data into a list of images & labels
- get just the labels from the test data
In [33]:
# Note: This might take a minute or so due to unravelling 790 batches
testDataLabels = []
for images, labels in testingData10p.unbatch(): # unbatch the test data and get images and labels
testDataLabels.append(labels.numpy().argmax()) # append the INDEX which has the largest value (labels are one-hot)In [34]:
# check what they look like
testDataLabels[:10]Out [34]:
In [35]:
len(testDataLabels)Out [35]:
The number of calculated labels matches the number of images.
Get an Accuracy Score On Prediction Results
In [36]:
sklearn_accuracy = accuracy_score(testDataLabels, pred_classes)
sklearn_accuracyOut [36]:
In [37]:
print(f"Close? {np.isclose(loaded_accuracy, sklearn_accuracy)} | Difference: {loaded_accuracy - sklearn_accuracy}")Visualize Predictions With a Confusion Matrix
In [40]:
# Note: The following confusion matrix code is a remix of Scikit-Learn's
# plot_confusion_matrix function - https://scikit-learn.org/stable/modules/generated/sklearn.metrics.plot_confusion_matrix.html
import itertools
import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import confusion_matrix
# Our function needs a different name to sklearn's plot_confusion_matrix
def make_confusion_matrix(y_true, y_pred, classes=None, figsize=(10, 10), text_size=15, norm=False, savefig=False):
"""Makes a labelled confusion matrix comparing predictions and ground truth labels.
If classes is passed, confusion matrix will be labelled, if not, integer class values
will be used.
Args:
y_true: Array of truth labels (must be same shape as y_pred).
y_pred: Array of predicted labels (must be same shape as y_true).
classes: Array of class labels (e.g. string form). If `None`, integer labels are used.
figsize: Size of output figure (default=(10, 10)).
text_size: Size of output figure text (default=15).
norm: normalize values or not (default=False).
savefig: save confusion matrix to file (default=False).
Returns:
A labelled confusion matrix plot comparing y_true and y_pred.
Example usage:
make_confusion_matrix(y_true=test_labels, # ground truth test labels
y_pred=y_preds, # predicted labels
classes=class_names, # array of class label names
figsize=(15, 15),
text_size=10)
"""
# Create the confustion matrix
cm = confusion_matrix(y_true, y_pred)
cm_norm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis] # normalize it
n_classes = cm.shape[0] # find the number of classes we're dealing with
# Plot the figure and make it pretty
fig, ax = plt.subplots(figsize=figsize)
cax = ax.matshow(cm, cmap=plt.cm.Blues) # colors will represent how 'correct' a class is, darker == better
fig.colorbar(cax)
# Are there a list of classes?
if classes:
labels = classes
else:
labels = np.arange(cm.shape[0])
# Label the axes
ax.set(title="Confusion Matrix",
xlabel="Predicted label",
ylabel="True label",
xticks=np.arange(n_classes), # create enough axis slots for each class
yticks=np.arange(n_classes),
xticklabels=labels, # axes will labeled with class names (if they exist) or ints
yticklabels=labels)
# Make x-axis labels appear on bottom
ax.xaxis.set_label_position("bottom")
ax.xaxis.tick_bottom()
### Added: Rotate xticks for readability & increase font size (required due to such a large confusion matrix)
plt.xticks(rotation=70, fontsize=text_size)
plt.yticks(fontsize=text_size)
# Set the threshold for different colors
threshold = (cm.max() + cm.min()) / 2.
# Plot the text on each cell
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
if norm:
plt.text(j, i, f"{cm[i, j]} ({cm_norm[i, j]*100:.1f}%)",
horizontalalignment="center",
color="white" if cm[i, j] > threshold else "black",
size=text_size)
else:
plt.text(j, i, f"{cm[i, j]}",
horizontalalignment="center",
color="white" if cm[i, j] > threshold else "black",
size=text_size)
# Save the figure to the current working directory
if savefig:
fig.savefig("confusion_matrix.png")In [41]:
testingClassNames = testingData10p.class_names
make_confusion_matrix(y_true=testDataLabels,
y_pred=pred_classes,
classes=testingClassNames,
figsize=(100, 100),
text_size=20,
norm=False,
savefig=True)
Get A Classification Report
The sklearn classification_report() outputs the precision, recall and f1-score's per class:
- Precision - Proportion of true positives over total number of samples. Higher precision leads to less false positives (model predicts 1 when it should've been 0).
- Recall - Proportion of true positives over total number of true positives and false negatives (model predicts 0 when it should've been 1). Higher recall leads to less false negatives.
- F1 score - Combines precision and recall into one metric. 1 is best, 0 is worst.
In [44]:
classificationReport = classification_report(testDataLabels, pred_classes)
print(classificationReport)Visualize Classification Prediction Scores By Class
Create Dictionary of Classifictaions By Index
In [45]:
# Get a dictionary of the classification report
classification_report_dict = classification_report(testDataLabels, pred_classes, output_dict=True)
classification_report_dictOut [45]:
Create Dictionary of Accuracy Scores By Class
In [48]:
# Create empty dictionary
class_f1_scores = {}
# Loop through classification report items
for k, v in classification_report_dict.items():
if k == "accuracy": # stop once we get to accuracy key
break
else:
# Append class names and f1-scores to new dictionary
class_f1_scores[testingClassNames[int(k)]] = v["f1-score"]
class_f1_scoresOut [48]:
Sort Classes By f-1 score descending
In [50]:
# Turn f1-scores into dataframe for visualization
f1_scores = pd.DataFrame({"class_name": list(class_f1_scores.keys()),
"f1-score": list(class_f1_scores.values())}).sort_values("f1-score", ascending=False)
f1_scores.head()Out [50]:
Horizontal Bar
In [52]:
fig, ax = plt.subplots(figsize=(12, 25))
scores = ax.barh(range(len(f1_scores)), f1_scores["f1-score"].values)
ax.set_yticks(range(len(f1_scores)))
ax.set_yticklabels(list(f1_scores["class_name"]))
ax.set_xlabel("f1-score")
ax.set_title("F1-Scores for 10 Different Classes")
ax.invert_yaxis(); # reverse the order
def autolabel(rects): # Modified version of: https://matplotlib.org/examples/api/barchart_demo.html
"""
Attach a text label above each bar displaying its height (it's value).
"""
for rect in rects:
width = rect.get_width()
ax.text(1.03*width, rect.get_y() + rect.get_height()/1.5,
f"{width:.2f}",
ha='center', va='bottom')
autolabel(scores)
Visualize Predictions on Test Images
Image Loader Function
In [53]:
def load_and_prep_image(filename, img_shape=224, scale=True):
# Read in the image
img = tf.io.read_file(filename)
# Decode it into a tensor
img = tf.io.decode_image(img)
# Resize the image
img = tf.image.resize(img, [img_shape, img_shape])
if scale:
# Rescale the image (get all values between 0 and 1)
return img/255.
else:
return imgVisualize A Few
In [74]:
plt.figure(figsize=(17, 10))
for i in range(3):
# Choose a random image from a random class
randomClassName = random.choice(testingClassNames)
randomFileName = random.choice(os.listdir(testDirPath + "/" + randomClassName))
randomFile = testDirPath + randomClassName + "/" + randomFileName
# Load the image and make predictions
img = load_and_prep_image(randomFile, scale=False) # don't scale images for EfficientNet predictions
pred_prob = m0.predict(tf.expand_dims(img, axis=0)) # model accepts tensors of shape [None, 224, 224, 3]
pred_class = testingClassNames[pred_prob.argmax()] # find the predicted class
# Plot the image(s)
plt.subplot(1, 3, i+1)
plt.imshow(img/255.)
if randomClassName == pred_class: # Change the color of text based on whether prediction is right or wrong
title_color = "g"
else:
title_color = "r"
plt.title(f"actual: {randomClassName} \n pred: {pred_class} \n prob: {pred_prob.max():.2f}", c=title_color)
plt.axis(False);
Finding & Visualizing Most-Wrong Predictions
- focus in on the wrong-"est" predictions
Data-Prep: Collect Test Image Paths
In [80]:
# 1. Get the filenames of all of our test data
testImageFilePaths = []
for filepath in testingData10p.list_files("101_food_classes_10_percent/test/*/*.jpg",
shuffle=False):
testImageFilePaths.append(filepath.numpy())
testImageFilePaths[:10]Out [80]:
Data-Prep: Build DataFrame With Prediction Values
In [94]:
predictionsDataFrame = pd.DataFrame({"img_path": testImageFilePaths,
"real_class_index": testDataLabels,
"predicted_classes": pred_classes,
"pred_conf": pred_probs.max(axis=1), # get the maximum prediction probability value
"real_classname": [testingClassNames[i] for i in testDataLabels],
"pred_classname": [testingClassNames[i] for i in pred_classes],
"pred_correct": testDataLabels == pred_classes,
})
predictionsDataFrame.head()Out [94]:
Data-Prep: Sort & Get most incorrect
In [113]:
# 4. Get the top 100 wrong examples
wrongSorted = predictionsDataFrame[predictionsDataFrame["pred_correct"] == False].sort_values("pred_conf", ascending=False)
# wrongSorted.head(20)
top_100_wrong = wrongSorted[:100]
top_100_wrong.head(20)Out [113]:
Visualize Most-Wrong Predictions
In [98]:
images_to_view = 9
start_index = 10 # change the start index to view more
plt.figure(figsize=(15, 10))
for i, row in enumerate(top_100_wrong[start_index:start_index+images_to_view].itertuples()):
plt.subplot(3, 3, i+1)
img = load_and_prep_image(row[1], scale=True)
_, _, _, _, pred_prob, y_true, y_pred, _ = row # only interested in a few parameters of each row
plt.imshow(img)
plt.title(f"actual: {y_true} \n pred: {y_pred} \n prob: {pred_prob:.2f}", loc="left")
plt.axis(False)
Group & Count Incorrect Classifications
In [138]:
top_250_wrong = wrongSorted[:250]
top_250_wrong.head(20)
justRealClasses = top_250_wrong[['real_classname', 'pred_classname']].copy()
mismatchedLabels = justRealClasses.groupby(['real_classname', 'pred_classname']).size().to_frame('size').sort_values(by="size", ascending=False)
mismatchedLabels.head(20)Out [138]:
Predict & Visualize on new images
Get Images
In [139]:
!wget https://storage.googleapis.com/ztm_tf_course/food_vision/custom_food_images.zip
unzip_data("custom_food_images.zip")Get Image file paths
In [143]:
custom_food_images_list = ["custom_food_images/" + img_path for img_path in os.listdir("custom_food_images")]
custom_food_images_listOut [143]:
Loop, Predict, and visualize
In [147]:
# Make predictions on custom food images
for img in custom_food_images_list:
img = load_and_prep_image(img, scale=False) # load in target image and turn it into tensor
pred_prob = m0.predict(tf.expand_dims(img, axis=0)) # make prediction on image with shape [None, 224, 224, 3]
pred_class = testingClassNames[pred_prob.argmax()] # find the predicted class label
# Plot the image with appropriate annotations
plt.figure()
plt.imshow(img/255.) # imshow() requires float inputs to be normalized
plt.title(f"pred: {pred_class} \n prob: {pred_prob.max():.2f}")
plt.axis(False)
