diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c
index 7272e248db87..693954f771ec 100644
--- a/mm/kasan/shadow.c
+++ b/mm/kasan/shadow.c
@@ -492,7 +492,8 @@ void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
 	 * Don't tag executable memory.
 	 * The kernel doesn't tolerate having the PC register tagged.
 	 */
-	if (!(flags & KASAN_VMALLOC_PROT_NORMAL))
+	if (IS_ENABLED(CONFIG_KASAN_SW_TAGS) &&
+	    !(flags & KASAN_VMALLOC_PROT_NORMAL))
 		return (void *)start;
 
 	start = set_tag(start, kasan_random_tag());