config: Read both top-level and network-subdir config files.

This lets you have a default, but also a network-specific config.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Changelog-changed: Options: `config` and <network>/`config` read by default.

common/configdir.c

@@ -79,7 +79,7 @@ static void config_log_stderr_exit(const char *fmt, ...)
 	errx(1, "%s", msg);
 }
 
-void parse_include(const char *filename, bool must_exist, bool early)
+static void parse_include(const char *filename, bool must_exist, bool early)
 {
 	char *contents, **lines;
 	char **all_args; /*For each line: either `--`argument, include file, or NULL*/
@@ -149,12 +149,12 @@ void parse_include(const char *filename, bool must_exist, bool early)
 	tal_free(contents);
 }
 
-static char *default_configdir(const tal_t *ctx)
+static char *default_base_configdir(const tal_t *ctx)
 {
 	char *path;
 	const char *env = getenv("HOME");
 	if (!env)
-		return tal_strdup(ctx, ".");
+		return path_cwd(ctx);
 
 	path = path_join(ctx, env, ".lightning");
 	return path;
@@ -203,6 +203,42 @@ void setup_option_allocators(void)
 	opt_set_alloc(opt_allocfn, tal_reallocfn, tal_freefn);
 }
 
+/* network is NULL for parsing top-level config file. */
+static void parse_implied_config_file(const char *config_dir,
+				      const char *network,
+				      bool early)
+{
+	const char *dir, *filename;
+
+	if (config_dir)
+		dir = path_join(NULL, take(path_cwd(NULL)), config_dir);
+	else
+		dir = default_base_configdir(NULL);
+
+	if (network)
+		dir = path_join(NULL, take(dir), network);
+
+	filename = path_join(NULL, take(dir), "config");
+	parse_include(filename, false, early);
+	tal_free(filename);
+}
+
+/* If they specify --conf, we just read that.
+ * Otherwise we read <lightning-dir>/config then <lightning-dir>/<network>/config
+ */
+void parse_config_files(const char *config_filename,
+			const char *config_dir,
+			bool early)
+{
+	if (config_filename) {
+		parse_include(config_filename, true, early);
+		return;
+	}
+
+	parse_implied_config_file(config_dir, NULL, early);
+	parse_implied_config_file(config_dir, chainparams->network_name, early);
+}
+
 void initial_config_opts(const tal_t *ctx,
 			 int argc, char *argv[],
 			 char **config_filename,
@@ -216,7 +252,14 @@ void initial_config_opts(const tal_t *ctx,
 	*config_filename = NULL;
 	opt_register_early_arg("--conf=<file>", opt_set_abspath, NULL,
 			       config_filename,
-			       "Specify configuration file (default: <lightning-dir>/config)");
+			       "Specify configuration file");
+
+	/* Cmdline can also set lightning-dir. */
+	*config_dir = NULL;
+	opt_register_early_arg("--lightning-dir=<dir>",
+			       opt_set_talstr, NULL,
+			       config_dir,
+			       "Set working directory. All other files are relative to this");
 
 	/* Handle --version (and exit) here too */
 	opt_register_version();
@@ -228,14 +271,22 @@ void initial_config_opts(const tal_t *ctx,
 
 	opt_register_early_arg("--conf=<file>", opt_ignore, NULL,
 			       config_filename,
-			       "Specify configuration file (default: <lightning-dir>/config)");
+			       "Specify configuration file");
+
+	/* If they set --conf it can still set --lightning-dir */
+	if (!*config_filename) {
+		opt_register_early_arg("--lightning-dir=<dir>",
+				       opt_ignore, opt_show_charp,
+				       config_dir,
+				       "Set working directory. All other files are relative to this");
+	} else {
+		opt_register_early_arg("--lightning-dir=<dir>",
+				       opt_set_talstr, NULL,
+				       config_dir,
+				       "Set working directory. All other files are relative to this");
+	}
 
 	/* Now, config file (or cmdline) can set network and lightning-dir */
-	*config_dir = NULL;
-	opt_register_early_arg("--lightning-dir=<dir>",
-			       opt_set_talstr, NULL,
-			       config_dir,
-			       "Set working directory. All other files are relative to this");
 
 	/* We need to know network early, so we can set defaults (which normal
 	 * options can change) and default config_dir */
@@ -256,6 +307,8 @@ void initial_config_opts(const tal_t *ctx,
 	/* Read config file first, since cmdline must override */
 	if (*config_filename)
 		parse_include(*config_filename, true, true);
+	else
+		parse_implied_config_file(*config_dir, NULL, true);
 	opt_early_parse_incomplete(argc, argv, opt_log_stderr_exit);
 
 	/* We use a global (in common/utils.h) for the chainparams.
@@ -264,7 +317,7 @@ void initial_config_opts(const tal_t *ctx,
 		chainparams = chainparams_for_network("testnet");
 
 	if (!*config_dir)
-		*config_dir = default_configdir(ctx);
+		*config_dir = default_base_configdir(ctx);
 
 	/* Make sure it's absolute */
 	*config_dir = path_join(ctx, take(path_cwd(NULL)), take(*config_dir));
@@ -274,7 +327,7 @@ void initial_config_opts(const tal_t *ctx,
 
 	opt_register_early_arg("--conf=<file>", opt_ignore, NULL,
 			       config_filename,
-			       "Specify configuration file (default: <lightning-dir>/config)");
+			       "Specify configuration file");
 	opt_register_early_arg("--lightning-dir=<dir>",
 			       opt_ignore, opt_show_charp,
 			       config_dir,

common/configdir.h

@@ -16,8 +16,12 @@ void initial_config_opts(const tal_t *ctx,
 			 char **config_dir,
 			 char **rpc_filename);
 
-/* Parse a specific include file */
+/* If they specify --conf, we just read that.
-void parse_include(const char *filename, bool must_exist, bool early);
+ * If they specify --lightning-dir, we just read <lightning_dir>/config.
+ * Otherwise, we read ../config (toplevel), and config (network-level) */
+void parse_config_files(const char *config_filename,
+			const char *config_dir,
+			bool early);
 
 /* For listconfigs to access. */
 char *opt_ignore(const char *arg, void *unused);

doc/lightningd-config.5

@@ -7,15 +7,16 @@ lightningd-config - Lightning daemon configuration file
 
 .SH DESCRIPTION
 
-When \fBlightningd\fR(8) starts up, it reads a configuration file\. By default
+When \fBlightningd\fR(8) starts up it usually reads a general configuration
-that is \fIconfig\fR in the \fB\.lightning\fR subdirectory of the home
+file (default: \fB$HOME/\.lightning/config\fR) then a network-specific
-directory (if it exists), but that can be changed by the
+configuration file (default: \fB$HOME/\.lightning/testnet/config\fR)\.  This can
-\fI--lightning-dir\fR or \fI--conf\fR options on the \fBlightningd\fR(8) command line\.
+be changed: see \fI--conf\fR and \fI--lightning-dir\fR\.
 
 
-Configuration file options are processed first, then command line
+General configuration files are processed first, then network-specific
-options: later options override earlier ones except \fIaddr\fR options 
+ones, then command line options: later options override earlier ones
-and \fIlog-level\fR with subsystems, which accumulate\.
+except \fIaddr\fR options and \fIlog-level\fR with subsystems, which
+accumulate\.
 
 
 \fIinclude \fR followed by a filename includes another configuration file at that
@@ -191,8 +192,8 @@ Run in the background, suppress stdout and stderr\.
 
 
  \fBconf\fR=\fIPATH\fR
-Sets configuration file (default: \fBlightning-dir\fR/\fIconfig\fR )\. If this
+Sets configuration file, and disable reading the normal general and network
-is a relative path, it is relative to the starting directory, not
+ones\. If this is a relative path, it is relative to the starting directory, not
 \fBlightning-dir\fR (unlike other paths)\. \fIPATH\fR must exist and be
 readable (we allow missing files in the default case)\. Using this inside
 a configuration file is meaningless\.

doc/lightningd-config.5.md

@@ -9,14 +9,15 @@ SYNOPSIS
 DESCRIPTION
 -----------
 
-When lightningd(8) starts up, it reads a configuration file. By default
+When lightningd(8) starts up it usually reads a general configuration
-that is *config* in the **.lightning** subdirectory of the home
+file (default: **$HOME/.lightning/config**) then a network-specific
-directory (if it exists), but that can be changed by the
+configuration file (default: **$HOME/.lightning/testnet/config**).  This can
-*--lightning-dir* or *--conf* options on the lightningd(8) command line.
+be changed: see *--conf* and *--lightning-dir*.
 
-Configuration file options are processed first, then command line
+General configuration files are processed first, then network-specific
-options: later options override earlier ones except *addr* options 
+ones, then command line options: later options override earlier ones
-and *log-level* with subsystems, which accumulate.
+except *addr* options and *log-level* with subsystems, which
+accumulate.
 
 *include * followed by a filename includes another configuration file at that
 point, relative to the current configuration file.
@@ -150,8 +151,8 @@ Set JSON-RPC socket (or /dev/tty), such as for lightning-cli(1).
 Run in the background, suppress stdout and stderr.
 
  **conf**=*PATH*
-Sets configuration file (default: **lightning-dir**/*config* ). If this
+Sets configuration file, and disable reading the normal general and network
-is a relative path, it is relative to the starting directory, not
+ones. If this is a relative path, it is relative to the starting directory, not
 **lightning-dir** (unlike other paths). *PATH* must exist and be
 readable (we allow missing files in the default case). Using this inside
 a configuration file is meaningless.

lightningd/options.c

@@ -629,25 +629,6 @@ static void check_config(struct lightningd *ld)
 		fatal("--always-use-proxy needs --proxy");
 }
 
-/**
- * We turn the config file into cmdline arguments. @early tells us
- * whether to parse early options only (and ignore any unknown ones),
- * or the non-early options.
- */
-static void opt_parse_from_config(struct lightningd *ld, bool early)
-{
-	const char *filename;
-
-	/* The default config doesn't have to exist, but if the config was
-	 * specified on the command line it has to exist. */
-	if (ld->config_filename != NULL)
-		filename = ld->config_filename;
-	else
-		filename = path_join(tmpctx, ld->config_dir, "config");
-
-	parse_include(filename, ld->config_filename != NULL, early);
-}
-
 static char *test_subdaemons_and_exit(struct lightningd *ld)
 {
 	test_subdaemons(ld);
@@ -965,9 +946,9 @@ void handle_early_opts(struct lightningd *ld, int argc, char *argv[])
 	 *  mimic this API here, even though they're on separate lines.*/
 	register_opts(ld);
 
-	/* Now look inside config file, but only handle the early
+	/* Now look inside config file(s), but only handle the early
 	 * options (testnet, plugins etc), others may be added on-demand */
-	opt_parse_from_config(ld, true);
+	parse_config_files(ld->config_filename, ld->config_dir, true);
 
 	/* Early cmdline options now override config file options. */
 	opt_early_parse_incomplete(argc, argv, opt_log_stderr_exit);
@@ -981,7 +962,7 @@ void handle_opts(struct lightningd *ld, int argc, char *argv[])
 	/* Now look for config file, but only handle non-early
 	 * options, early ones have been parsed in
 	 * handle_early_opts */
-	opt_parse_from_config(ld, false);
+	parse_config_files(ld->config_filename, ld->config_dir, false);
 
 	/* Now parse cmdline, which overrides config. */
 	opt_parse(&argc, argv, opt_log_stderr_exit);

tests/test_misc.py

@@ -1782,3 +1782,15 @@ def test_include(node_factory):
     l1.start()
 
     assert l1.rpc.listconfigs('alias')['alias'] == 'conf2'
+
+
+def test_config_in_subdir(node_factory):
+    l1 = node_factory.get_node(start=False)
+
+    subdir = os.path.join(l1.daemon.opts.get("lightning-dir"), "regtest")
+    os.makedirs(subdir)
+    with open(os.path.join(subdir, "config"), 'w') as f:
+        f.write('alias=test_config_in_subdir')
+    l1.start()
+
+    assert l1.rpc.listconfigs('alias')['alias'] == 'test_config_in_subdir'