Verify plug state after every automated switch instead of trusting the command succeeded

solixauto/engine.py

@@ -319,6 +319,49 @@ class Engine:
             self.report(f"FAILED to turn {self._word(desired)}: {type(err).__name__}: {err}")
             return False
 
+        await asyncio.sleep(1.0)
+        try:
+            confirmed = await self.target.get_state(session)
+        except Exception:
+            confirmed = None
+
+        if confirmed is None:
+            self.report(
+                f"sent {self._word(desired)} but could not read back the plug's "
+                "actual state to confirm it took effect"
+            )
+            await self.notify_switch_problem(
+                "unconfirmed", desired, None, rule, variables, reason
+            )
+            self.record_event(
+                None,
+                f"{reason} (sent {self._word(desired)}, could not verify)",
+                rule,
+                variables,
+                cause="unverified",
+            )
+            return False
+
+        if confirmed != desired:
+            self.report(
+                f"WARNING: commanded {self._word(desired)} ({reason}) but the "
+                f"plug reports {self._word(confirmed)}. Not treating this as "
+                "applied; will retry."
+            )
+            self.last_commanded = confirmed
+            await self.notify_switch_problem(
+                "failed", desired, confirmed, rule, variables, reason
+            )
+            self.record_event(
+                confirmed,
+                f"{reason} (commanded {self._word(desired)}, plug stayed "
+                f"{self._word(confirmed)})",
+                rule,
+                variables,
+                cause="switch_failed",
+            )
+            return False
+
         self.last_commanded = desired
         self.last_action_at = now
         self.recent_actions.append(now)
@@ -328,6 +371,32 @@ class Engine:
         await self.notify(desired, rule, variables, event="action")
         return True
 
+    async def notify_switch_problem(
+        self, kind, desired, confirmed, rule, variables, reason
+    ):
+        if not self.notifier.available():
+            self.report(
+                f"switch {kind} but no notification channel is enabled", force=True
+            )
+            return
+
+        settings = self.profile.notifications
+        context = self.context(confirmed, rule, variables, "switch_" + kind)
+        context["reason"] = reason
+        context["attempted"] = self._word(desired)
+        context["actual"] = self._word(confirmed) if confirmed is not None else "unknown"
+
+        template = (
+            settings.switch_unconfirmed_template
+            if kind == "unconfirmed"
+            else settings.switch_failed_template
+        )
+
+        body = render(template, context)
+        title = render(settings.title or "{profile}", context)
+
+        await self.notifier.send(title, body, priority="urgent", key="switch_" + kind)
+
     @staticmethod
     def _word(desired):
         return "ON" if desired else "OFF"
@@ -1215,6 +1284,7 @@ def _preview_notification(profile, rule, variables, desired):
             "target_host": target_profile.get("access", {}).get("host", ""),
             "target_channel": profile.target_channel or 0,
             "time": stamp(),
+            "clock": clock_stamp(),
             "event": "action",
         }
     )

solixauto/monitor.py

@@ -1040,6 +1040,8 @@ PAGE = """<!doctype html>
 
   .event-cause.floor { color: var(--alert); border-color: currentColor; }
   .event-cause.stale { color: var(--solar); border-color: currentColor; }
+  .event-cause.switch_failed { color: var(--alert); border-color: currentColor; }
+  .event-cause.unverified { color: var(--alert); border-color: currentColor; }
 
   .event-detail {
     font-family: ui-monospace, "SF Mono", Menlo, monospace;
@@ -1529,7 +1531,9 @@ const CAUSE_LABEL = {
   floor: 'safety floor',
   stale: 'telemetry lost',
   recovered: 'recovered',
-  manual: 'manual'
+  manual: 'manual',
+  switch_failed: 'switch did not take effect',
+  unverified: 'switch unverified'
 };
 
 function dayLabel(epoch) {

solixauto/rules.py

@@ -206,6 +206,20 @@ class NotificationSettings:
             raw.get("recovered_template")
             or "{clock} {source_name}: telemetry is back after {outage}. Rules resumed."
         )
+        self.switch_failed_template = str(
+            raw.get("switch_failed_template")
+            or (
+                "{clock} {source_name}: commanded {target_name} {attempted}, but "
+                "it still reports {actual}. Will keep retrying. ({reason})"
+            )
+        )
+        self.switch_unconfirmed_template = str(
+            raw.get("switch_unconfirmed_template")
+            or (
+                "{clock} {source_name}: sent {target_name} {attempted}, but could "
+                "not read back its state to confirm. ({reason})"
+            )
+        )
         self.title = str(raw.get("title") or "{profile}")
         self.throttle = parse_duration(
             raw.get("throttle", DEFAULT_NOTIFY_THROTTLE), "notifications.throttle"
@@ -529,7 +543,14 @@ def validate(profile):
                 f"{sorted(configured)} are enabled in notifications.yaml"
             )
 
-        templates_to_check = [profile.notifications.template, profile.notifications.title]
+        templates_to_check = [
+            profile.notifications.template,
+            profile.notifications.title,
+            profile.notifications.stale_template,
+            profile.notifications.recovered_template,
+            profile.notifications.switch_failed_template,
+            profile.notifications.switch_unconfirmed_template,
+        ]
         for rule in profile.active_rules():
             if rule.notify_template:
                 templates_to_check.append(rule.notify_template)
@@ -538,6 +559,7 @@ def validate(profile):
             "profile", "rule", "action", "action_word", "source_name", "source_model",
             "source_serial", "target_name", "target_model", "target_host",
             "target_channel", "condition", "time", "clock", "reason", "event",
+            "last_seen", "outage", "attempted", "actual",
         }
         for template in templates_to_check:
             for field in notify_module.template_fields(template):