arm/lib: Improve FVPRunner shutdown logic

We have encountered intermittent hanging during FVP shutdown, so improve
the termination logic by first issuing a terminate(), waiting a bit
then, if necessary, issuing a kill().

Move returncode logic to after the telnet/pexpect cleanup so it
actually runs.

Move pexpect.EOF logic into FVPRunner.stop so that it executes before
closing the pexpect handle.

Issue-Id: SCM-4957
Signed-off-by: Peter Hoyes <Peter.Hoyes@arm.com>
Change-Id: Iebb3c3c89367256b1e116e66ffdb6b742358bce4
Signed-off-by: Jon Mason <jon.mason@arm.com>

meta-arm/lib/fvp/runner.py

@@ -72,24 +72,36 @@ class FVPRunner:
 
     async def stop(self):
         if self._fvp_process:
-            self._logger.debug(f"Killing FVP PID {self._fvp_process.pid}")
+            self._logger.debug(f"Terminating FVP PID {self._fvp_process.pid}")
             try:
                 self._fvp_process.terminate()
+                await asyncio.wait_for(self._fvp_process.wait(), 10.0)
+            except asyncio.TimeoutError:
+                self._logger.debug(f"Killing FVP PID {self._fvp_process.pid}")
+                self._fvp_process.kill()
             except ProcessLookupError:
                 pass
 
-            if await self._fvp_process.wait() != 0:
-                self._logger.info(f"FVP quit with code {self._fvp_process.returncode}")
-                return self._fvp_process.returncode
-            else:
-                return 0
-
         for telnet in self._telnets:
-            await telnet.terminate()
-            await telnet.wait()
+            try:
+                telnet.terminate()
+                await asyncio.wait_for(telnet.wait(), 10.0)
+            except asyncio.TimeoutError:
+                telnet.kill()
+            except ProcessLookupError:
+                pass
 
-        for pexpect in self._pexpects:
-            pexpect.close()
+        for console in self._pexpects:
+            import pexpect
+            # Ensure pexpect logs all remaining output to the logfile
+            console.expect(pexpect.EOF, timeout=5.0)
+            console.close()
+
+        if self._fvp_process and self._fvp_process.returncode:
+            self._logger.info(f"FVP quit with code {self._fvp_process.returncode}")
+            return self._fvp_process.returncode
+        else:
+            return 0
 
     async def run(self, until=None):
         if until and until():

meta-arm/lib/oeqa/controllers/fvp.py

@@ -127,12 +127,6 @@ class OEFVPSerialTarget(OEFVPSSHTarget):
                 default_test_file = f"{name}_log{self.test_log_suffix}"
                 os.symlink(default_test_file, self.bootlog)
 
-    async def _after_stop(self):
-        # Ensure pexpect logs all remaining output to the logfile
-        for terminal in self.terminals.values():
-            terminal.expect(pexpect.EOF, timeout=5)
-            terminal.close()
-
     def _get_terminal(self, name):
         return self.terminals[name]