diff --git a/scripts/build-image-if-changed.sh b/scripts/build-image-if-changed.sh
index 5f52652..ff04eda 100755
--- a/scripts/build-image-if-changed.sh
+++ b/scripts/build-image-if-changed.sh
@@ -50,7 +50,18 @@ fingerprint() {
 candidate_fingerprint="$(fingerprint "$candidate_tag")"
 docker pull "$PRIMARY_TAG" >/dev/null 2>&1 || true
 current_fingerprint="$(docker image inspect "$PRIMARY_TAG" --format '{{ index .Config.Labels "org.ajpanton.content-fingerprint" }}' 2>/dev/null || true)"
-if [[ -z "$current_fingerprint" ]] && docker image inspect "$PRIMARY_TAG" >/dev/null 2>&1; then
+current_labels_match=true
+for oci_label in "${oci_labels[@]}"; do
+    label_key="${oci_label%%=*}"
+    label_value="${oci_label#*=}"
+    current_label_value="$(docker image inspect "$PRIMARY_TAG" --format "{{ index .Config.Labels \"${label_key}\" }}" 2>/dev/null || true)"
+    if [[ "$current_label_value" != "$label_value" ]]; then
+        current_labels_match=false
+        break
+    fi
+done
+
+if [[ "$current_labels_match" == true && -z "$current_fingerprint" ]] && docker image inspect "$PRIMARY_TAG" >/dev/null 2>&1; then
     current_fingerprint="$(fingerprint "$PRIMARY_TAG")"
 fi