Re: [PATCH v11 8/9] hugetlb_cgroup: Add hugetlb_cgroup reservation tests

[Mina Almasry]

On Tue, Feb 4, 2020 at 8:26 AM Sandipan Das <sandipan@xxxxxxxxxxxxx> wrote:
>
>
> There are still a couple of places where 2MB page size is being used.
> These are my workarounds to get the tests running on ppc64.
>

Thanks for the changes!

> Also I had missed running charge_reserved_hugetlb.sh the last time.
> Right now, it stops at the following scenario.
>
> Test normal case with write.
> private=, populate=, method=2, reserve=
> nr hugepages = 10
> writing cgroup limit: 83886080
> writing reseravation limit: 83886080
>
> Starting:
> hugetlb_usage=0
> reserved_usage=0
> expect_failure is 0
> Putting task in cgroup 'hugetlb_cgroup_test'
> Method is 2
> Executing ./write_to_hugetlbfs -p /mnt/huge/test -s 83886080 -w  -m 2  -l
> Writing to this path: /mnt/huge/test
> Writing this size: 83886080
> Not populating.
> Using method=2
> Shared mapping.
> RESERVE mapping.
> Allocating using SHM.
> shmid: 0x5, shmget key:0
> shmaddr: 0x7dfffb000000
> Writing to memory.
> Starting the writes:
> .write_result is 0
> .After write:
> hugetlb_usage=16777216
> reserved_usage=83886080
> ....kiling write_to_hugetlbfs
> ...Received 2.
> Deleting the memory
> Done deleting the memory
> 16777216
> 83886080
> Memory charged to hugtlb=16777216
> Memory charged to reservation=83886080
> expected (83886080) != actual (16777216): Reserved memory charged to hugetlb cgroup.
> CLEANUP DONE
>
>

So the problem in this log seems to be that this log line is missing:
    echo Waiting for hugetlb memory to reach size $size.

The way the test works is that it starts a process that writes the
hugetlb memory, then it *should* wait until the memory is written,
then it should record the cgroup accounting and kill the process. It
seems from your log that the wait doesn't happen, so the test
continues before the background process has had time to write the
memory properly. Essentially wait_for_hugetlb_memory_to_get_written()
never gets called in your log.

Can you try this additional attached diff on top of your changes? I
attached the diff and pasted the same here, hopefully one works for
you:

diff --git a/tools/testing/selftests/vm/charge_reserved_hugetlb.sh
b/tools/testing/selftests/vm/charge_reserved_hugetlb.sh
index efd68093ce3e9..18d33684faade 100755
--- a/tools/testing/selftests/vm/charge_reserved_hugetlb.sh
+++ b/tools/testing/selftests/vm/charge_reserved_hugetlb.sh
@@ -169,19 +169,36 @@ function write_hugetlbfs_and_get_usage() {
   echo reserved_usage="$reserved_before"
   echo expect_failure is "$expect_failure"

+  output=$(mktemp)
   set +e
   if [[ "$method" == "1" ]] || [[ "$method" == 2 ]] ||
     [[ "$private" == "-r" ]] && [[ "$expect_failure" != 1 ]]; then

     bash write_hugetlb_memory.sh "$size" "$populate" "$write" \
-      "$cgroup" "$path" "$method" "$private" "-l" "$reserve" &
+      "$cgroup" "$path" "$method" "$private" "-l" "$reserve" 2>&1 |
tee $output &

     local write_result=$?
+    local write_pid=$!

-    if [[ "$reserve" != "-n" ]]; then
-      wait_for_hugetlb_memory_to_get_reserved "$cgroup" "$size"
-    elif [[ "$populate" == "-o" ]] || [[ "$write" == "-w" ]]; then
+    until grep -q -i "DONE" $output; do
+      echo waiting for DONE signal.
+      if ! ps $write_pid > /dev/null
+      then
+        echo "FAIL: The write died"
+        cleanup
+        exit 1
+      fi
+      sleep 0.5
+    done
+
+    echo ================= write_hugetlb_memory.sh output is:
+    cat $output
+    echo ================= end output.
+
+    if [[ "$populate" == "-o" ]] || [[ "$write" == "-w" ]]; then
       wait_for_hugetlb_memory_to_get_written "$cgroup" "$size"
+    elif [[ "$reserve" != "-n" ]]; then
+      wait_for_hugetlb_memory_to_get_reserved "$cgroup" "$size"
     else
       # This case doesn't produce visible effects, but we still have
       # to wait for the async process to start and execute...
@@ -227,7 +244,7 @@ function cleanup_hugetlb_memory() {
   set +e
   local cgroup="$1"
   if [[ "$(pgrep -f write_to_hugetlbfs)" != "" ]]; then
-    echo kiling write_to_hugetlbfs
+    echo killing write_to_hugetlbfs
     killall -2 write_to_hugetlbfs
     wait_for_hugetlb_memory_to_get_depleted $cgroup
   fi
diff --git a/tools/testing/selftests/vm/write_to_hugetlbfs.c
b/tools/testing/selftests/vm/write_to_hugetlbfs.c
index 85811c3384a10..7f75ad5f7b580 100644
--- a/tools/testing/selftests/vm/write_to_hugetlbfs.c
+++ b/tools/testing/selftests/vm/write_to_hugetlbfs.c
@@ -207,13 +207,13 @@ int main(int argc, char **argv)
  }
  printf("shmid: 0x%x, shmget key:%d\n", shmid, key);

- shmaddr = shmat(shmid, NULL, 0);
- if (shmaddr == (char *)-1) {
+ ptr = shmat(shmid, NULL, 0);
+ if (ptr == (int *)-1) {
  perror("Shared memory attach failure");
  shmctl(shmid, IPC_RMID, NULL);
  exit(2);
  }
- printf("shmaddr: %p\n", shmaddr);
+ printf("shmaddr: %p\n", ptr);

  break;
  default:
@@ -223,25 +223,7 @@ int main(int argc, char **argv)

  if (write) {
  printf("Writing to memory.\n");
- if (method != SHM) {
- memset(ptr, 1, size);
- } else {
- printf("Starting the writes:\n");
- for (i = 0; i < size; i++) {
- shmaddr[i] = (char)(i);
- if (!(i % (1024 * 1024)))
- printf(".");
- }
- printf("\n");
-
- printf("Starting the Check...");
- for (i = 0; i < size; i++)
- if (shmaddr[i] != (char)i) {
- printf("\nIndex %lu mismatched\n", i);
- exit(3);
- }
- printf("Done.\n");
- }
+ memset(ptr, 1, size);
  }

  if (want_sleep) {
@@ -253,7 +235,7 @@ int main(int argc, char **argv)
  sleep(100);
  }

- switch (method == HUGETLBFS) {
+ if (method == HUGETLBFS) {
  close(fd);
  }
diff --git a/tools/testing/selftests/vm/charge_reserved_hugetlb.sh b/tools/testing/selftests/vm/charge_reserved_hugetlb.sh
index efd68093ce3e9..18d33684faade 100755
--- a/tools/testing/selftests/vm/charge_reserved_hugetlb.sh
+++ b/tools/testing/selftests/vm/charge_reserved_hugetlb.sh
@@ -169,19 +169,36 @@ function write_hugetlbfs_and_get_usage() {
   echo reserved_usage="$reserved_before"
   echo expect_failure is "$expect_failure"

+  output=$(mktemp)
   set +e
   if [[ "$method" == "1" ]] || [[ "$method" == 2 ]] ||
     [[ "$private" == "-r" ]] && [[ "$expect_failure" != 1 ]]; then

     bash write_hugetlb_memory.sh "$size" "$populate" "$write" \
-      "$cgroup" "$path" "$method" "$private" "-l" "$reserve" &
+      "$cgroup" "$path" "$method" "$private" "-l" "$reserve" 2>&1 | tee $output &

     local write_result=$?
+    local write_pid=$!

-    if [[ "$reserve" != "-n" ]]; then
-      wait_for_hugetlb_memory_to_get_reserved "$cgroup" "$size"
-    elif [[ "$populate" == "-o" ]] || [[ "$write" == "-w" ]]; then
+    until grep -q -i "DONE" $output; do
+      echo waiting for DONE signal.
+      if ! ps $write_pid > /dev/null
+      then
+        echo "FAIL: The write died"
+        cleanup
+        exit 1
+      fi
+      sleep 0.5
+    done
+
+    echo ================= write_hugetlb_memory.sh output is:
+    cat $output
+    echo ================= end output.
+
+    if [[ "$populate" == "-o" ]] || [[ "$write" == "-w" ]]; then
       wait_for_hugetlb_memory_to_get_written "$cgroup" "$size"
+    elif [[ "$reserve" != "-n" ]]; then
+      wait_for_hugetlb_memory_to_get_reserved "$cgroup" "$size"
     else
       # This case doesn't produce visible effects, but we still have
       # to wait for the async process to start and execute...
@@ -227,7 +244,7 @@ function cleanup_hugetlb_memory() {
   set +e
   local cgroup="$1"
   if [[ "$(pgrep -f write_to_hugetlbfs)" != "" ]]; then
-    echo kiling write_to_hugetlbfs
+    echo killing write_to_hugetlbfs
     killall -2 write_to_hugetlbfs
     wait_for_hugetlb_memory_to_get_depleted $cgroup
   fi
diff --git a/tools/testing/selftests/vm/write_to_hugetlbfs.c b/tools/testing/selftests/vm/write_to_hugetlbfs.c
index 85811c3384a10..7f75ad5f7b580 100644
--- a/tools/testing/selftests/vm/write_to_hugetlbfs.c
+++ b/tools/testing/selftests/vm/write_to_hugetlbfs.c
@@ -207,13 +207,13 @@ int main(int argc, char **argv)
 		}
 		printf("shmid: 0x%x, shmget key:%d\n", shmid, key);

-		shmaddr = shmat(shmid, NULL, 0);
-		if (shmaddr == (char *)-1) {
+		ptr = shmat(shmid, NULL, 0);
+		if (ptr == (int *)-1) {
 			perror("Shared memory attach failure");
 			shmctl(shmid, IPC_RMID, NULL);
 			exit(2);
 		}
-		printf("shmaddr: %p\n", shmaddr);
+		printf("shmaddr: %p\n", ptr);

 		break;
 	default:
@@ -223,25 +223,7 @@ int main(int argc, char **argv)

 	if (write) {
 		printf("Writing to memory.\n");
-		if (method != SHM) {
-			memset(ptr, 1, size);
-		} else {
-			printf("Starting the writes:\n");
-			for (i = 0; i < size; i++) {
-				shmaddr[i] = (char)(i);
-				if (!(i % (1024 * 1024)))
-					printf(".");
-			}
-			printf("\n");
-
-			printf("Starting the Check...");
-			for (i = 0; i < size; i++)
-				if (shmaddr[i] != (char)i) {
-					printf("\nIndex %lu mismatched\n", i);
-					exit(3);
-				}
-			printf("Done.\n");
-		}
+		memset(ptr, 1, size);
 	}

 	if (want_sleep) {
@@ -253,7 +235,7 @@ int main(int argc, char **argv)
 			sleep(100);
 	}

-	switch (method == HUGETLBFS) {
+	if (method == HUGETLBFS) {
 		close(fd);
 	}