diff --git a/pkg/steps/input_image_tag.go b/pkg/steps/input_image_tag.go
index 1e009d8e5f..d2e1bfd5ae 100644
--- a/pkg/steps/input_image_tag.go
+++ b/pkg/steps/input_image_tag.go
@@ -108,7 +108,7 @@ func (s *inputImageTagStep) run(ctx context.Context) error {
 		},
 		Tag: &imagev1.TagReference{
 			ReferencePolicy: imagev1.TagReferencePolicy{
-				Type: imagev1.SourceTagReferencePolicy,
+				Type: imagev1.LocalTagReferencePolicy,
 			},
 			From: from,
 			ImportPolicy: imagev1.TagImportPolicy{
diff --git a/pkg/steps/input_image_tag_test.go b/pkg/steps/input_image_tag_test.go
index 453a9eeda2..bc2c061bb2 100644
--- a/pkg/steps/input_image_tag_test.go
+++ b/pkg/steps/input_image_tag_test.go
@@ -120,7 +120,7 @@ func TestInputImageTagStep(t *testing.T) {
 				ImportMode: imagev1.ImportModePreserveOriginal,
 			},
 			ReferencePolicy: imagev1.TagReferencePolicy{
-				Type: imagev1.SourceTagReferencePolicy,
+				Type: imagev1.LocalTagReferencePolicy,
 			},
 		},
 	}
@@ -233,7 +233,7 @@ func TestInputImageTagStepExternal(t *testing.T) {
 				ImportMode: imagev1.ImportModePreserveOriginal,
 			},
 			ReferencePolicy: imagev1.TagReferencePolicy{
-				Type: imagev1.SourceTagReferencePolicy,
+				Type: imagev1.LocalTagReferencePolicy,
 			},
 		},
 	}
diff --git a/pkg/steps/source.go b/pkg/steps/source.go
index ac18768008..5f44e269d9 100644
--- a/pkg/steps/source.go
+++ b/pkg/steps/source.go
@@ -435,6 +435,11 @@ func handleFailedBuild(ctx context.Context, client BuildClient, ns, name string,
 		return err
 	}
 
+	if isNonRetriableInfraError(b.Status.LogSnippet) {
+		logrus.Infof("Build %s failed with non-retriable infrastructure error (%s), will not be retried", name, b.Status.Reason)
+		return err
+	}
+
 	logrus.Infof("Build %s previously failed from an infrastructure error (%s), retrying...", name, b.Status.Reason)
 
 	// Remove workload from metrics watching since we're about to delete and recreate the build
@@ -632,6 +637,10 @@ func isInfraReason(reason buildapi.StatusReason) bool {
 	return false
 }
 
+func isNonRetriableInfraError(logSnippet string) bool {
+	return strings.Contains(logSnippet, "manifest unknown")
+}
+
 func hintsAtInfraReason(logSnippet string) bool {
 	return strings.Contains(logSnippet, "error: build error: no such image") ||
 		strings.Contains(logSnippet, "[Errno 256] No more mirrors to try.") ||