[PATCH] selftests/prctl: Fix non-anonymous VMA mapping in set-anon-vma-name test

From: Hongfu Li

Date: Mon Aug 03 2026 - 06:39:26 EST


From: Hongfu Li <lihongfu@xxxxxxxxxx>

The test creates a non-anonymous VMA (ptr_not_anon) via mmap() with
MAP_PRIVATE but without MAP_ANONYMOUS, using fd=0 (stdin) as the file
descriptor. This always fails because fd=0 is not a regular file,
and the failure was hidden because ASSERT_NE() incorrectly checked
for NULL instead of MAP_FAILED.

Fix by using mkstemp() + ftruncate() to create a real temporary file,
then mapping it with MAP_PRIVATE to obtain a genuine file-backed VMA.
Also fix the mmap() error checks to use MAP_FAILED instead of NULL,
and pass fd=-1 for the anonymous mapping for clarity. The temp file
is unlinked immediately so it does not persist on disk.

Signed-off-by: Hongfu Li <lihongfu@xxxxxxxxxx>
---
.../selftests/prctl/set-anon-vma-name-test.c | 21 ++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/tools/testing/selftests/prctl/set-anon-vma-name-test.c b/tools/testing/selftests/prctl/set-anon-vma-name-test.c
index ac6721b184a6..5f9589534da8 100644
--- a/tools/testing/selftests/prctl/set-anon-vma-name-test.c
+++ b/tools/testing/selftests/prctl/set-anon-vma-name-test.c
@@ -4,11 +4,13 @@
*/

#include <errno.h>
+#include <fcntl.h>
#include <sys/prctl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <string.h>
+#include <unistd.h>

#include "kselftest_harness.h"

@@ -73,15 +75,24 @@ int was_renaming_successful(char *target_name, unsigned long ptr)

FIXTURE(vma) {
void *ptr_anon, *ptr_not_anon;
+ int fd_not_anon;
};

FIXTURE_SETUP(vma) {
+ char template[] = "./set-anon-vma-test-XXXXXX";
+
self->ptr_anon = mmap(NULL, AREA_SIZE, PROT_READ | PROT_WRITE,
- MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
- ASSERT_NE(self->ptr_anon, NULL);
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ ASSERT_NE(self->ptr_anon, MAP_FAILED);
+
+ self->fd_not_anon = mkstemp(template);
+ ASSERT_NE(self->fd_not_anon, -1);
+ unlink(template);
+ ASSERT_EQ(ftruncate(self->fd_not_anon, AREA_SIZE), 0);
self->ptr_not_anon = mmap(NULL, AREA_SIZE, PROT_READ | PROT_WRITE,
- MAP_PRIVATE, 0, 0);
- ASSERT_NE(self->ptr_not_anon, NULL);
+ MAP_PRIVATE, self->fd_not_anon, 0);
+ ASSERT_NE(self->ptr_not_anon, MAP_FAILED);
+ close(self->fd_not_anon);
}

FIXTURE_TEARDOWN(vma) {
@@ -98,7 +109,7 @@ TEST_F(vma, renaming) {
EXPECT_EQ(rename_vma((unsigned long)self->ptr_anon, AREA_SIZE, BAD_NAME), -EINVAL);

TH_LOG("Try to rename non-anonymous VMA");
- EXPECT_EQ(rename_vma((unsigned long) self->ptr_not_anon, AREA_SIZE, GOOD_NAME), -EINVAL);
+ EXPECT_EQ(rename_vma((unsigned long) self->ptr_not_anon, AREA_SIZE, GOOD_NAME), -EBADF);
}

TEST_HARNESS_MAIN
--
2.54.0