lightningd: clean up subds before freeing HTLCs.

Otherwise we get weird effects, as htlcs are being freed:

```
2022-01-26T05:07:37.8774610Z lightningd-1: 2022-01-26T04:47:48.770Z DEBUG   030eeb52087b9dbb27b7aec79ca5249369f6ce7b20a5684ce38d9f4595a21c2fda-chan#8: Failing HTLC 18446744073709551615 due to peer death
2022-01-26T05:07:37.8775287Z lightningd-1: 2022-01-26T04:47:48.770Z **BROKEN** 030eeb52087b9dbb27b7aec79ca5249369f6ce7b20a5684ce38d9f4595a21c2fda-chan#8: Neither origin nor in?
```

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

lightningd/lightningd.c

@@ -529,6 +529,11 @@ static void shutdown_subdaemons(struct lightningd *ld)
 	ld->gossip = subd_shutdown(ld->gossip, 10);
 	ld->hsm = subd_shutdown(ld->hsm, 10);
 
+	/*~ Closing the hsmd means all other subdaemons should be exiting;
+	 * deal with that cleanly before we start freeing internal
+	 * structures. */
+	subd_shutdown_remaining(ld);
+
 	/* Now we free all the HTLCs */
 	free_htlcs(ld, NULL);
 

lightningd/onchain_control.c

@@ -570,10 +570,15 @@ static void onchain_error(struct channel *channel,
 			  bool warning UNUSED,
 			  const u8 *err_for_them UNUSED)
 {
+	channel_set_owner(channel, NULL);
+
+	/* This happens on shutdown: fine */
+	if (channel->peer->ld->state == LD_STATE_SHUTDOWN)
+		return;
+
 	/* FIXME: re-launch? */
 	log_broken(channel->log, "%s", desc);
 	channel_set_billboard(channel, true, desc);
-	channel_set_owner(channel, NULL);
 }
 
 /* With a reorg, this can get called multiple times; each time we'll kill

lightningd/opening_common.c

@@ -105,6 +105,8 @@ void uncommitted_channel_disconnect(struct uncommitted_channel *uc,
 {
 	u8 *msg = towire_connectd_peer_disconnected(tmpctx, &uc->peer->id);
 	log_(uc->log, level, NULL, false, "%s", desc);
+	/* NULL when we're shutting down */
+	if (uc->peer->ld->connectd)
 		subd_send_msg(uc->peer->ld->connectd, msg);
 	if (uc->fc && uc->fc->cmd)
 		was_pending(command_fail(uc->fc->cmd, LIGHTNINGD, "%s", desc));
@@ -191,6 +193,8 @@ void handle_reestablish(struct lightningd *ld,
 				      "Unknown channel for reestablish");
 		log_debug(ld->log, "Reestablish on UNKNOWN channel %s",
 			  type_to_string(tmpctx, struct channel_id, channel_id));
+		/* Unless we're shutting down */
+		if (ld->connectd)
 			subd_send_msg(ld->connectd,
 				      take(towire_connectd_peer_final_msg(NULL, peer_id,
 									  err)));

lightningd/subd.c

@@ -895,6 +895,20 @@ struct subd *subd_shutdown(struct subd *sd, unsigned int seconds)
 	return tal_free(sd);
 }
 
+void subd_shutdown_remaining(struct lightningd *ld)
+{
+	struct subd *subd;
+
+	/* We give them a second to finish exiting, before we kill
+	 * them in destroy_subd() */
+	sleep(1);
+
+	while ((subd = list_top(&ld->subds, struct subd, list)) != NULL) {
+		/* Destructor removes from list */
+		io_close(subd->conn);
+	}
+}
+
 void subd_release_channel(struct subd *owner, const void *channel)
 {
 	/* If owner is a per-peer-daemon, and not already freeing itself... */

lightningd/subd.h

@@ -222,6 +222,15 @@ void subd_release_channel(struct subd *owner, const void *channel);
  */
 struct subd *subd_shutdown(struct subd *subd, unsigned int seconds);
 
+/**
+ * subd_shutdown_remaining - kill all remaining (per-peer) subds
+ * @ld: lightningd
+ *
+ * They should already be exiting (since we shutdown hsmd), but
+ * make sure they have.
+ */
+void subd_shutdown_remaining(struct lightningd *ld);
+
 /* Ugly helper to get full pathname of the current binary. */
 const char *find_my_abspath(const tal_t *ctx, const char *argv0);