Improve on randomized tests flakiness (#2823)

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

dockerfiles/ci/buster/php-8.3/suppr.txt

@@ -3,3 +3,4 @@ leak:timer_create
 leak:add_to_global
 leak:_dl_map_object_deps
 leak:__res_context_send
+leak:_dl_make_tlsdesc_dynamic

tests/randomized/analyze-results.php

@@ -10,14 +10,23 @@ function analyze_web($tmpScenariosFolder)
     $unexpectedCodes = [];
     $possibleSegfaults = [];
     $minimumRequestCount = [];
+    $prepareErrors = [];
 
     foreach (scandir($resultsFolder) as $identifier) {
         if (in_array($identifier, ['.', '..'])) {
             continue;
         }
+
+        $analyzed[] = $identifier;
+
         $scenarioResultsRoot = $resultsFolder . DIRECTORY_SEPARATOR . $identifier;
+        $prepareErrorFilePath = $scenarioResultsRoot . DIRECTORY_SEPARATOR . 'prepare-error';
         $absFilePath = $scenarioResultsRoot . DIRECTORY_SEPARATOR . 'results.json';
-        $analyzed[] = $identifier;
+
+        if (file_exists($prepareErrorFilePath)) {
+            $prepareErrors[$identifier] = (int)file_get_contents($prepareErrorFilePath);
+            continue;
+        }
 
         $jsonResult = json_decode(file_get_contents($absFilePath), 1);
 
@@ -71,6 +80,10 @@ function analyze_web($tmpScenariosFolder)
         echo "Minimum request not matched: " . var_export($minimumRequestCount, 1) . "\n";
         $isError = true;
     }
+    if (count($prepareErrors)) {
+        echo "Prepare step failed with error: " . var_export($prepareErrors, 1) . "\n";
+        $isError = count($prepareErrors) > count($analyzed) / 2; // In that case it's probably not transient
+    }
 
     if ($isError) {
         return false;
@@ -115,7 +128,13 @@ function analyze_cli($tmpScenariosFolder)
 
         $analyzed[] = $identifier;
 
-        $absFilePath = $resultsFolder . DIRECTORY_SEPARATOR . $identifier . DIRECTORY_SEPARATOR . 'memory.out';
+        $scenarioResultsRoot = $resultsFolder . DIRECTORY_SEPARATOR . $identifier;
+        $absFilePath = $scenarioResultsRoot . DIRECTORY_SEPARATOR . 'memory.out';
+        $prepareErrorFilePath = $scenarioResultsRoot . DIRECTORY_SEPARATOR . 'prepare-error';
+
+        if (file_exists($prepareErrorFilePath)) {
+            continue; // We already handled it in analyze_web
+        }
 
         $values = array_map('intval', array_filter(
             file($absFilePath),

tests/randomized/docker/buster.Dockerfile

@@ -10,6 +10,9 @@ RUN for DIR in /opt/php/*; do (echo "zend_extension=opcache.so"; echo "opcache.e
 # don't execute an asan *binary* under qemu
 RUN mv /opt/php/debug/bin/php-config /opt/php/debug/bin/php-config-debug; cp /opt/php/debug-zts-asan/bin/php-config /opt/php/debug/bin/php-config
 
+# install redis for randomized tests
+RUN echo "extension=redis" >> $(php-config --ini-dir)/redis.ini
+
 # Igbinary
 RUN set -eux; \
     pecl install "igbinary"; \
@@ -65,6 +68,10 @@ RUN sed -i 's/apache2/httpd/' /etc/apache2/envvars
 ADD run.sh /scripts/run.sh
 ADD prepare.sh /scripts/prepare.sh
 
+# actually bind the sidecar error output to docker out
+ENV _DD_DEBUG_SIDECAR_LOG_METHOD=file:///proc/1/fd/2
+ENV DD_SPAWN_WORKER_USE_EXEC=1
+
 WORKDIR /var/www/html
 
 ENV COMPOSER_CACHE_DIR /composer-cache

tests/randomized/docker/centos.Dockerfile

@@ -30,7 +30,7 @@ RUN set -eux; \
 
 # Redis
 RUN set -eux; \
-    printf 'yes' | pecl install "redis"; \
+    printf 'yes' | pecl install "redis$(if [ ${PHP_VERSION/./} -le 71 ]; then echo -5.3.7; fi)"; \
     for DIR in /opt/php/*; do echo "extension=redis.so" > $DIR/conf.d/redis.ini; done
 
 # Create coredumps folder
@@ -83,6 +83,8 @@ RUN echo "CoreDumpDirectory /tmp/corefiles" >> /etc/httpd/conf/httpd.conf
 ADD run.sh /scripts/run.sh
 ADD prepare.sh /scripts/prepare.sh
 
+ENV DD_SPAWN_WORKER_USE_EXEC=1
+
 WORKDIR /var/www/html
 
 ENV COMPOSER_CACHE_DIR /composer-cache

tests/randomized/docker/prepare.sh

@@ -25,7 +25,11 @@ php -v
 echo "Starting PHP-FPM"
 mkdir -p /var/log/php-fpm/
 chmod a+w /var/log/php-fpm/
-php-fpm -D -d datadog.trace.log_file=/results/dd_php_error.log
+if ldd $(which php) 2>/dev/null | grep -q libasan; then
+  php-fpm -D -d datadog.trace.log_file=/results/dd_php_error.log
+else
+  nohup strace -ttfs 200 bash -c 'php-fpm -F -d datadog.trace.log_file=/results/dd_php_error.log 2>&3' 0<&- 3>&2 2>/results/php-fpm.strace >/dev/null &
+fi
 sleep 1
 
 # Start nginx

tests/randomized/docker/run.sh

@@ -3,12 +3,23 @@
 set -e
 
 . /scripts/enable-coredump.sh
-bash /scripts/prepare.sh
+
+ret=0
+bash /scripts/prepare.sh || ret=$?
+if [[ $ret -ne 0 ]]; then
+  # handle transient prepare failures
+  echo $ret > /results/prepare-error
+  exit $ret
+fi
 
 echo "Starting web load"
 vegeta -cpus=2 attack -format=http -targets=/vegeta-request-targets.txt -duration=${DURATION:-30s} -keepalive=false -max-workers=10 -workers=10 -rate=100 | tee results.bin | vegeta report --type=json --output=/results/results.json
 echo "Done web load"
 
 echo "Starting CLI load"
-sh /cli-runner.sh
+if ldd $(which php) 2>/dev/null | grep -q libasan; then
+  sh /cli-runner.sh
+else
+  strace -ttfs 200 bash -c 'sh /cli-runner.sh 2>&3' 3>&2 2>/results/php-cli.strace
+fi
 echo "Done CLI load"