Keep agent lanes alive after controller response failures

static/agent/ec-agent.py

@@ -12,6 +12,7 @@ import contextlib
 from datetime import datetime, timezone
 import math
 import hashlib
+import http.client
 import fcntl
 import json
 import os
@@ -32,7 +33,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.5.1"
+VERSION = "1.5.2"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1099,7 +1100,7 @@ class Agent:
                     raise AgentError("Controller redirects are not allowed")
             with urllib.request.build_opener(NoRedirect()).open(req, timeout=35) as response:
                 return json.load(response)
-        except (OSError, ValueError):
+        except (OSError, ValueError, http.client.HTTPException):
             raise AgentError("Controller request failed") from None
 
     def cpu_usage_percent(self):
@@ -2354,6 +2355,8 @@ class Agent:
                 self.heartbeat()
             except (AgentError, OSError, ValueError):
                 pass
+            except Exception as err:
+                print(f"Heartbeat recovered from {type(err).__name__}", file=sys.stderr)
 
     @staticmethod
     def read_operation(operation):
@@ -2421,6 +2424,9 @@ class Agent:
                         self.stopping.wait(0.25)
                 except (AgentError, OSError, ValueError, KeyError, sqlite3.Error):
                     self.stopping.wait(3)
+                except Exception as err:
+                    print(f"{lane} lane recovered from {type(err).__name__}", file=sys.stderr)
+                    self.stopping.wait(3)
         finally:
             journal.close()
 
@@ -2475,6 +2481,9 @@ class Agent:
                 break
             except (AgentError, OSError, ValueError):
                 self.stopping.wait(3)
+            except Exception as err:
+                print(f"Initial heartbeat recovered from {type(err).__name__}", file=sys.stderr)
+                self.stopping.wait(3)
         heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
         heartbeats.start()
         # Preserve two concurrent health checks even while files are being read.
@@ -2488,6 +2497,8 @@ class Agent:
             except (AgentError, OSError, ValueError, KeyError):
                 # Logs deliberately omit URLs, tokens, payloads and raw errors.
                 print("Host controller contact unavailable; existing containers continue running", file=sys.stderr)
+            except Exception as err:
+                print(f"Write lane recovered from {type(err).__name__}", file=sys.stderr)
             self.stopping.wait(0.25 if completed else 3)
         for reader in readers:
             reader.join(timeout=2)