QA-536: move san file handling in its own class

js/client/modules/@arangodb/testutils/instance.js

@@ -33,6 +33,7 @@ const rp = require('@arangodb/testutils/result-processing');
 const yaml = require('js-yaml');
 const internal = require('internal');
 const crashUtils = require('@arangodb/testutils/crash-utils');
+const {sanHandler} = require('@arangodb/testutils/san-file-handler');
 const crypto = require('@arangodb/crypto');
 const ArangoError = require('@arangodb').ArangoError;
 const debugGetFailurePoints = require('@arangodb/test-helper').debugGetFailurePoints;
@@ -41,7 +42,6 @@ const debugGetFailurePoints = require('@arangodb/test-helper').debugGetFailurePo
 const {
   toArgv,
   executeExternal,
-  executeExternalAndWait,
   killExternal,
   statusExternal,
   statisticsExternal,
@@ -73,13 +73,6 @@ let tcpdump;
 
 let PORTMANAGER;
 
-var regex = /[^\u0000-\u00ff]/; // Small performance gain from pre-compiling the regex
-function containsDoubleByte(str) {
-    if (!str.length) return false;
-    if (str.charCodeAt(0) > 255) return true;
-    return regex.test(str);
-}
-
 function getSockStatFile(pid) {
   try {
     return fs.read("/proc/" + pid + "/net/sockstat");
@@ -226,9 +219,7 @@ class instance {
     }
     this.JWT = null;
     this.jwtFiles = null;
-
-    this.sanOptions = _.clone(this.options.sanOptions);
-    this.sanFiles = [];
+    this.sanHandler = new sanHandler('arangod', this.options.sanOptions, this.options.isSan, this.options.extremeVerbosity);
 
     this._makeArgsArangod();
 
@@ -472,17 +463,7 @@ class instance {
     if (this.args.hasOwnProperty('server.jwt-secret')) {
       this.JWT = this.args['server.jwt-secret'];
     }
-    if (this.options.isSan) {
-      let rootDir = this.rootDir;
-      if (containsDoubleByte(rootDir)) {
-        rootDir = this.topLevelTmpDir;
-      }
-      for (const [key, value] of Object.entries(this.sanOptions)) {
-        let oneLogFile = fs.join(rootDir, key.toLowerCase().split('_')[0] + '.log');
-        this.sanOptions[key]['log_path'] = oneLogFile;
-        this.sanFiles.push(oneLogFile);
-      }
-    }
+    this.sanHandler.detectLogfiles(this.rootDir, this.topLevelTmpDir);
   }
 
   // //////////////////////////////////////////////////////////////////////////////
@@ -553,7 +534,7 @@ class instance {
 
   cleanup() {
     if ((this.pid !== null) && (this.exitStatus === null)) {
-      print(RED + "killing instance (again?) to make sure we can delete its files!" + RESET);
+      print(RED + Date() + "killing instance (again?) to make sure we can delete its files!" + RESET);
       this.terminateInstance();
     }
     if (this.options.extremeVerbosity) {
@@ -694,20 +675,9 @@ class instance {
     if (this.options.extremeVerbosity) {
       print(Date() + ' starting process ' + cmd + ' with arguments: ' + JSON.stringify(argv));
     }
-    let backup = {};
-    if (this.options.isSan) {
-      for (const [key, value] of Object.entries(this.sanOptions)) {
-        let oneSet = "";
-        for (const [keyOne, valueOne] of Object.entries(value)) {
-          if (oneSet.length > 0) {
-            oneSet += ":";
-          }
-          oneSet += `${keyOne}=${valueOne}`;
-        }
-        backup[key] = process.env[key];
-        process.env[key] = oneSet;
-      }
-    }
+
+    this.sanHandler.setSanOptions();
+
     if ((this.useableMemory === undefined) && (this.options.memory !== undefined)){
       throw new Error(`${this.name} don't have planned memory though its configured!`);
     }
@@ -719,11 +689,8 @@ class instance {
     }
     process.env['ARANGODB_SERVER_DIR'] = this.rootDir;
     let ret = executeExternal(cmd, argv, false, pu.coverageEnvironment());
-    if (this.options.isSan) {
-      for (const [key, value] of Object.entries(backup)) {
-        process.env[key] = value;
-      }
-    }
+
+    this.sanHandler.resetSanOptions();
     if (this.useableMemory !== 0) {
       delete process.env['ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY'];
     }
@@ -748,7 +715,7 @@ class instance {
 
     if (crashUtils.isEnabledWindowsMonitor(this.options, this, this.pid, pu.ARANGOD_BIN)) {
       if (!crashUtils.runProcdump(this.options, this, this.coreDirectory, this.pid)) {
-        print('Killing ' + pu.ARANGOD_BIN + ' - ' + JSON.stringify(this.args));
+        print(Date() + 'Killing ' + pu.ARANGOD_BIN + ' - ' + JSON.stringify(this.args));
         let res = killExternal(this.pid);
         this.pid = res.pid;
         this.exitStatus = res;
@@ -780,7 +747,7 @@ class instance {
     }
     if (crashUtils.isEnabledWindowsMonitor(this.options, this, this.pid, pu.ARANGOD_BIN)) {
       if (!crashUtils.runProcdump(this.options, this, this.coreDirectory, this.pid)) {
-        print('Killing ' + pu.ARANGOD_BIN + ' - ' + JSON.stringify(this.args));
+        print(Date() + 'Killing ' + pu.ARANGOD_BIN + ' - ' + JSON.stringify(this.args));
         let res = killExternal(this.pid);
         this.pid = res.pid;
         this.exitStatus = res;
@@ -793,25 +760,6 @@ class instance {
       internal.addPidToMonitor(this.pid);
     }
   };
-
-  fetchSanFileAfterExit() {
-    if (this.options.isSan) {
-      this.sanFiles.forEach(fileName => {
-        let fn = `${fileName}.arangod.${this.pid}`;
-        if (this.options.extremeVerbosity) {
-          print(`checking for ${fn}: ${fs.exists(fn)}`);
-        }
-        if (fs.exists(fn)) {
-          let content = fs.read(fn);
-          if (content.length > 10) {
-            crashUtils.GDB_OUTPUT += `Report of '${this.name}' in ${fn} contains: \n`;
-            crashUtils.GDB_OUTPUT += content;
-            this.serverCrashedLocal = true;
-          }
-        }
-      });
-    }
-  }
   waitForExitAfterDebugKill() {
     // Crashutils debugger kills our instance, but we neet to get
     // testing.js sapwned-PID-monitoring adjusted.
@@ -827,7 +775,7 @@ class instance {
     } catch(ex) {
       print(ex);
     }
-    this.fetchSanFileAfterExit();
+    this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
     this.pid = null;
     print('done');
   }
@@ -838,10 +786,10 @@ class instance {
     }
     this.exitStatus = statusExternal(this.pid, true);
     if (this.exitStatus.status !== 'TERMINATED') {
-      this.fetchSanFileAfterExit();
+      this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
       throw new Error(this.name + " didn't exit in a regular way: " + JSON.stringify(this.exitStatus));
     }
-    this.fetchSanFileAfterExit();
+    this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
     this.exitStatus = null;
     this.pid = null;
   }
@@ -1040,13 +988,13 @@ class instance {
     if ((this.exitStatus === null) ||
         (this.exitStatus.status === 'RUNNING')) {
       if (forceTerminate) {
-        let sockStat = this.getSockStat("Force killing - sockstat before: ");
+        let sockStat = this.getSockStat(Date() + "Force killing - sockstat before: ");
         this.killWithCoreDump('shutdown timeout; instance forcefully KILLED because of fatal timeout in testrun ' + sockStat);
         this.pid = null;
       } else if (this.options.useKillExternal) {
         let sockStat = this.getSockStat("Shutdown by kill - sockstat before: ");
         this.exitStatus = killExternal(this.pid);
-        this.fetchSanFileAfterExit();
+        this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
         this.pid = null;
         print(sockStat);
       } else if (this.protocol === 'unix') {
@@ -1068,7 +1016,7 @@ class instance {
           print(Date() + ' Wrong shutdown response: ' + JSON.stringify(reply) + "' " + sockStat + " continuing with hard kill!");
           this.shutdownArangod(true);
         } else {
-          this.fetchSanFileAfterExit();
+          this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
           if (!this.options.noStartStopLogs) {
             print(sockStat);
           }
@@ -1096,7 +1044,7 @@ class instance {
           this.shutdownArangod(true);
         }
         else {
-          this.fetchSanFileAfterExit();
+          this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
           if (!this.options.noStartStopLogs) {
             print(sockStat);
           }

js/client/modules/@arangodb/testutils/process-utils.js

@@ -32,6 +32,7 @@ const rp = require('@arangodb/testutils/result-processing');
 const yaml = require('js-yaml');
 const internal = require('internal');
 const crashUtils = require('@arangodb/testutils/crash-utils');
+const {sanHandler} = require('@arangodb/testutils/san-file-handler');
 const crypto = require('@arangodb/crypto');
 const ArangoError = require('@arangodb').ArangoError;
 const debugGetFailurePoints = require('@arangodb/test-helper').debugGetFailurePoints;
@@ -465,8 +466,8 @@ function setupBinaries (builddir, buildType, configDir) {
     let fileName = san + "_arangodb_suppressions.txt";
     if (!process.env.hasOwnProperty(envName) &&
         fs.exists(fileName)) {
-      // print('preparing ' + san + ' environment');
       process.env[envName] = `suppressions=${fs.join(fs.makeAbsolute(''), fileName)}`;
+      print('preparing ' + san + ' environment:', envName + '=' + process.env[envName]);
     }
   });
 
@@ -844,7 +845,7 @@ function executeAndWait (cmd, args, options, valgrindTest, rootDir, coreCheck =
       getStructure: function() { return {}; }
     };
   }
-
+  let sh;
   let res = {};
   if (platform.substr(0, 3) === 'win' && !options.disableMonitor) {
     res = executeExternal(cmd, args, false, coverageEnvironment());
@@ -876,14 +877,23 @@ function executeAndWait (cmd, args, options, valgrindTest, rootDir, coreCheck =
     }
   } else {
     // V8 executeExternalAndWait thinks that timeout is in ms, so *1000
-    res = executeExternalAndWait(cmd, args, false, timeout*1000, coverageEnvironment());
+    sh = new sanHandler(cmd.replace(/.*\//, ''), options.sanOptions, options.isSan, options.extremeVerbosity);
+    sh.detectLogfiles(instanceInfo.rootDir, instanceInfo.rootDir);
+    sh.setSanOptions();
+    res = executeExternalAndWait(cmd, args, false, timeout * 1000, coverageEnvironment());
     instanceInfo.pid = res.pid;
     instanceInfo.exitStatus = res;
+    sh.resetSanOptions();
     crashUtils.calculateMonitorValues(options, instanceInfo, res.pid, cmd);
   }
   const deltaTime = time() - startTime;
 
   let errorMessage = ' - ';
+  if (sh.fetchSanFileAfterExit(res.pid)) {
+    serverCrashedLocal = true;
+    res.status = false;
+    errorMessage += " Sanitizer indicated issues  - ";
+  }
 
   if (coreCheck &&
       instanceInfo.exitStatus.hasOwnProperty('signal') &&
@@ -945,7 +955,7 @@ function executeAndWait (cmd, args, options, valgrindTest, rootDir, coreCheck =
       duration: deltaTime
     };
   } else if (res.status === 'TIMEOUT') {
-    print('Killing ' + cmd + ' - ' + JSON.stringify(args));
+    print('Date() + Killing ' + cmd + ' - ' + JSON.stringify(args));
     let resKill = killExternal(res.pid, abortSignal);
     if (coreCheck) {
       print(Date() + " executeAndWait: Marking crashy because of timeout - " + JSON.stringify(instanceInfo));

js/client/modules/@arangodb/testutils/san-file-handler.js

@@ -0,0 +1,119 @@
+/* jshint strict: false, sub: true */
+/* global print, arango */
+'use strict';
+
+// //////////////////////////////////////////////////////////////////////////////
+// / DISCLAIMER
+// /
+// / Copyright 2014-2024 ArangoDB GmbH, Cologne, Germany
+// / Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+// /
+// / Licensed under the Business Source License 1.1 (the "License");
+// / you may not use this file except in compliance with the License.
+// / You may obtain a copy of the License at
+// /
+// /     https://github.com/arangodb/arangodb/blob/devel/LICENSE
+// /
+// / Unless required by applicable law or agreed to in writing, software
+// / distributed under the License is distributed on an "AS IS" BASIS,
+// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// / See the License for the specific language governing permissions and
+// / limitations under the License.
+// /
+// / Copyright holder is ArangoDB GmbH, Cologne, Germany
+// /
+// / @author Wilfried Goesgens
+// //////////////////////////////////////////////////////////////////////////////
+
+const _ = require('lodash');
+const fs = require('fs');
+const crashUtils = require('@arangodb/testutils/crash-utils');
+
+var regex = /[^\u0000-\u00ff]/; // Small performance gain from pre-compiling the regex
+function containsDoubleByte(str) {
+    if (!str.length) return false;
+    if (str.charCodeAt(0) > 255) return true;
+    return regex.test(str);
+}
+
+class sanHandler {
+  constructor(binaryName, sanOptions, isSan, extremeVerbosity) {
+    this.binaryName = binaryName;
+    this.sanOptions = _.clone(sanOptions);
+    this.enabled = isSan;
+    this.extremeVerbosity = extremeVerbosity;
+    this.sanitizerLogPaths = {};
+    this.backup = {};
+  }
+  detectLogfiles(rootDir, tmpDir) {
+    if (this.enabled) {
+      if (containsDoubleByte(rootDir)) {
+        rootDir = tmpDir;
+      }
+      for (const [key, value] of Object.entries(this.sanOptions)) {
+        let oneLogFile = fs.join(rootDir, key.toLowerCase().split('_')[0] + '.log');
+        // we need the log files to contain the exe name, otherwise our code to pick them up won't find them
+        this.sanOptions[key]['log_exe_name'] = "true";
+        const origPath = this.sanOptions[key]['log_path'];
+        this.sanOptions[key]['log_path'] = oneLogFile;
+        this.sanitizerLogPaths[key] = { upstream: origPath, local: oneLogFile };
+      }
+    }
+  }
+  setSanOptions() {
+    if (this.enabled) {
+      print("Using sanOptions ", this.sanOptions);
+      for (const [key, value] of Object.entries(this.sanOptions)) {
+        let oneSet = "";
+        for (const [keyOne, valueOne] of Object.entries(value)) {
+          if (oneSet.length > 0) {
+            oneSet += ":";
+          }
+          let val = valueOne.replaceAll(',', '_');
+          oneSet += `${keyOne}=${val}`;
+        }
+        this.backup[key] = process.env[key];
+        // print(`${key} => ${oneSet}`);
+        process.env[key] = oneSet;
+      }
+    }
+  }
+  resetSanOptions() {
+    if (this.enabled) {
+      for (const [key, value] of Object.entries(this.backup)) {
+        process.env[key] = value;
+      }
+    }
+  }
+
+  fetchSanFileAfterExit(pid) {
+    if (!this.enabled) {
+      return false;
+    }
+    let ret = false;
+    for (const [key, value] of Object.entries(this.sanitizerLogPaths)) {
+      print("processing ", value);
+      const { upstream, local } = value;
+      let fn = `${local}.${this.binaryname}.${pid}`;
+      if (this.extremeVerbosity) {
+        print(`checking for ${fn}: ${fs.exists(fn)}`);
+      }
+      if (fs.exists(fn)) {
+        let content = fs.read(fn);
+        if (upstream) {
+          print("found file ", fn, " - writing file ", `${upstream}.${this.binaryName}.${this.pid}`);
+          fs.write(`${upstream}.${this.binaryName}.${this.pid}`, content);
+        }
+        if (content.length > 10) {
+          crashUtils.GDB_OUTPUT += `Report of '${this.name}' in ${fn} contains: \n`;
+          crashUtils.GDB_OUTPUT += content;
+          ret = true;
+        }
+      }
+    }
+    return ret;
+  }
+
+}
+
+exports.sanHandler = sanHandler;