lightningd: switch parsing to common/configvar

Now we wire in the code which gathers configvars and parses from there; lightningd keeps the array of configuration variables for future use. Note that lightning-cli also needs to read the config, but it has its own options (including short ones!) and doesn't want to use this configvar mechanism, so we have a different API for that now. Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
2026-02-23 15:04:19 +01:00 · 2023-06-02 12:05:51 +09:30
parent f54e4f3084
commit edbaa944da
20 changed files with 312 additions and 320 deletions
--- a/common/configdir.c
+++ b/common/configdir.c
@@ -4,17 +4,21 @@
 #include <ccan/cast/cast.h>
 #include <ccan/err/err.h>
 #include <ccan/opt/opt.h>
+#include <ccan/opt/private.h>
 #include <ccan/tal/grab_file/grab_file.h>
 #include <ccan/tal/path/path.h>
 #include <ccan/tal/str/str.h>
 #include <common/configdir.h>
+#include <common/configvar.h>
 #include <common/utils.h>
 #include <common/version.h>

 bool deprecated_apis = true;
+int opt_exitcode = 1;

 /* The regrettable globals */
 static const tal_t *options_ctx;
+static struct configvar *current_cv;

 /* Override a tal string; frees the old one. */
 char *opt_set_talstr(const char *arg, char **p)
@@ -50,113 +54,55 @@ static void tal_freefn(void *ptr)
 	tal_free(ptr);
 }

-static int config_parse_line_number;
-
-static void config_log_stderr_exit(const char *fmt, ...)
-{
-	char *msg;
-	va_list ap;
-
-	va_start(ap, fmt);
-
-	/* This is the format we expect:*/
-	if (streq(fmt, "%s: %.*s: %s")) {
-		const char *argv0 = va_arg(ap, const char *);
-		unsigned int len = va_arg(ap, unsigned int);
-		const char *arg = va_arg(ap, const char *);
-		const char *problem = va_arg(ap, const char *);
-
-		assert(argv0 != NULL);
-		assert(arg != NULL);
-		assert(problem != NULL);
-		/*mangle it to remove '--' and add the line number.*/
-		msg = tal_fmt(NULL, "%s line %d: %.*s: %s",
-			      argv0,
-			      config_parse_line_number, len-2, arg+2, problem);
-	} else {
-		msg = tal_vfmt(NULL, fmt, ap);
-	}
-	va_end(ap);
-
-	errx(1, "%s", msg);
-}
-
-static void parse_include(const char *filename, bool must_exist, bool early,
-			  size_t depth)
+static struct configvar **gather_file_configvars(const tal_t *ctx,
+						 enum configvar_src src,
+						 const char *filename,
+						 bool must_exist,
+						 size_t include_depth)
 {
 	char *contents, **lines;
-	char **all_args; /*For each line: either `--`argument, include file, or NULL*/
-	char *argv[3];
-	int i, argc;
+	struct configvar **cvs = tal_arr(ctx, struct configvar *, 0);

-	contents = grab_file(NULL, filename);
+	contents = grab_file(tmpctx, filename);

 	/* The default config doesn't have to exist, but if the config was
 	 * specified on the command line it has to exist. */
 	if (!contents) {
 		if (must_exist)
 			err(1, "Opening and reading %s", filename);
-		return;
+		return cvs;
 	}

+	/* Break into lines. */
 	lines = tal_strsplit(contents, contents, "\r\n", STR_EMPTY_OK);
+	for (size_t i = 0; i < tal_count(lines) - 1; i++) {
+		/* Comments & blank lines*/
+		if (strstarts(lines[i], "#") || streq(lines[i], ""))
+			continue;

-	/* We have to keep all_args around, since opt will point into it: use
-	 * magic tal name to tell memleak this isn't one. */
-	all_args = tal_arr_label(options_ctx, char *, tal_count(lines) - 1,
-				 TAL_LABEL(options_array_notleak, ""));
+		if (strstarts(lines[i], "include ")) {
+			const char *included = lines[i] + strlen("include ");
+			struct configvar **sub;
+
+			if (include_depth > 100)
+				errx(1, "Include loop with %s and %s", filename, included);

-	for (i = 0; i < tal_count(lines) - 1; i++) {
-		if (strstarts(lines[i], "#")) {
-			all_args[i] = NULL;
-		} else if (strstarts(lines[i], "include ")) {
 			/* If relative, it's relative to current config file */
-			all_args[i] = path_join(all_args,
-						take(path_dirname(NULL,
-								  filename)),
-						lines[i] + strlen("include "));
-		} else {
-			/* Only valid forms are "foo" and "foo=bar" */
-			all_args[i] = tal_fmt(all_args, "--%s", lines[i]);
-		}
-		/* This isn't a leak either */
-		if (all_args[i])
-			tal_set_name(all_args[i], TAL_LABEL(config_notleak, ""));
-	}
-
-	/*
-	For each line we construct a fake argc,argv commandline.
-	argv[1] is the only element that changes between iterations.
-	*/
-	argc = 2;
-	argv[0] = cast_const(char *, filename);
-	argv[argc] = NULL;
-
-	for (i = 0; i < tal_count(all_args); i++) {
-		if (all_args[i] == NULL)
-			continue;
-
-		if (!strstarts(all_args[i], "--")) {
-			/* There could be more, but this gives a hint. */
-			if (depth > 100)
-				errx(1, "Include loop with %s and %s",
-				     filename, all_args[i]);
-			parse_include(all_args[i], true, early, ++depth);
+			sub = gather_file_configvars(NULL,
+						     src,
+						     path_join(tmpctx,
+							       take(path_dirname(NULL, filename)),
+							       included),
+						     true,
+						     include_depth + 1);
+			cvs = configvar_join(ctx, take(cvs), take(sub));
 			continue;
 		}

-		config_parse_line_number = i + 1;
-		argv[1] = all_args[i];
-		if (early) {
-			opt_early_parse_incomplete(argc, argv,
-						   config_log_stderr_exit);
-		} else {
-			opt_parse(&argc, argv, config_log_stderr_exit);
-			argc = 2; /* opt_parse might have changed it  */
-		}
+		tal_arr_expand(&cvs,
+			       configvar_new(cvs, src, filename, i+1, lines[i]));
 	}
-
-	tal_free(contents);
+	return cvs;
 }

 static char *default_base_configdir(const tal_t *ctx)
@@ -179,6 +125,12 @@ static char *opt_set_network(const char *arg, void *unused)
 {
 	assert(arg != NULL);

+	/* Ignore if called directly from opt (e.g. lightning-cli) */
+	if (!current_cv)
+		return NULL;
+
+	if (current_cv->src == CONFIGVAR_NETWORK_CONF)
+		return "not permitted in network-specific configuration files";
 	/* Set the global chainparams instance */
 	chainparams = chainparams_for_network(arg);
 	if (!chainparams)
@@ -197,46 +149,27 @@ static bool opt_show_network(char *buf, size_t len, const void *unused)
 	return true;
 }

-/* We track where we're getting options from, so we can detect misuse */
-enum parse_state {
-	CMDLINE = 1,
-	FORCED_CONFIG = 2,
-	TOPLEVEL_CONFIG = 4,
-	NETWORK_CONFIG = 8,
-};
-static enum parse_state parse_state = CMDLINE;
-
-static char *opt_restricted_cmdline(const char *arg, const void *unused)
+static char *opt_set_config_filename(const char *arg, char **p)
 {
-	if (parse_state != CMDLINE)
-		return "not permitted in configuration files";
-	return NULL;
+	/* Ignore if called directly from opt (e.g. lightning-cli) */
+	if (!current_cv)
+		return NULL;
+
+	if (current_cv->src == CONFIGVAR_CMDLINE)
+		return opt_set_abspath(arg, p);
+	return "not permitted in configuration files";
 }

-static char *opt_restricted_toplevel_noarg(const void *unused)
+static char *opt_set_lightning_dir(const char *arg, char **p)
 {
-	if (parse_state == NETWORK_CONFIG)
-		return "not permitted in network-specific configuration files";
-	return NULL;
-}
+	/* Ignore if called directly from opt (e.g. lightning-cli) */
+	if (!current_cv)
+		return NULL;

-static char *opt_restricted_toplevel(const char *arg, const void *unused)
-{
-	return opt_restricted_toplevel_noarg(NULL);
-}
-
-static char *opt_restricted_forceconf_only(const char *arg, const void *unused)
-{
-	if (parse_state != CMDLINE && parse_state != FORCED_CONFIG)
-		return "not permitted in implicit configuration files";
-	return NULL;
-}
-
-bool is_restricted_ignored(const void *fn)
-{
-	return fn == opt_restricted_toplevel_noarg
-		|| fn == opt_restricted_toplevel
-		|| fn == opt_restricted_forceconf_only;
+	if (current_cv->src == CONFIGVAR_CMDLINE
+	    || current_cv->src == CONFIGVAR_EXPLICIT_CONF)
+		return opt_set_abspath(arg, p);
+	return "not permitted in implicit configuration files";
 }

 void setup_option_allocators(void)
@@ -245,103 +178,155 @@ void setup_option_allocators(void)
 	opt_set_alloc(opt_allocfn, tal_reallocfn, tal_freefn);
 }

-/* network is NULL for parsing top-level config file. */
-static void parse_implied_config_file(const char *config_basedir,
-				      const char *network,
-				      bool early)
+static void parse_configvars(struct configvar **cvs,
+			     bool early,
+			     bool full_knowledge)
 {
-	const char *dir, *filename;
+	for (size_t i = 0; i < tal_count(cvs); i++) {
+		const char *problem;
+		bool should_know;

-	if (config_basedir)
-		dir = path_join(NULL, take(path_cwd(NULL)), config_basedir);
-	else
-		dir = default_base_configdir(NULL);
+		should_know = full_knowledge;
+		/* We should always know cmdline args in final parse */
+		if (!early && cvs[i]->src == CONFIGVAR_CMDLINE)
+			should_know = true;

-	if (network)
-		dir = path_join(NULL, take(dir), network);
+		current_cv = cvs[i];
+		problem = configvar_parse(cvs[i],
+					  early,
+					  should_know,
+					  IFDEV(true, false));
+		current_cv = NULL;
+		if (!problem)
+			continue;

-	filename = path_join(NULL, take(dir), "config");
-	parse_include(filename, false, early, 0);
-	tal_free(filename);
+		if (cvs[i]->file) {
+			errx(opt_exitcode, "Config file %s line %u: %s: %s",
+			     cvs[i]->file, cvs[i]->linenum,
+			     cvs[i]->configline, problem);
+		} else {
+			errx(opt_exitcode, "--%s: %s", cvs[i]->configline, problem);
+		}
+	}
 }

-/* If they specify --conf, we just read that.
- * Otherwise we read <lightning-dir>/config then <lightning-dir>/<network>/config
- */
-void parse_config_files(const char *config_filename,
-			const char *config_basedir,
-			bool early)
+static void finished_arg(int *argc, char **argv, size_t *idx,
+			 bool remove_args)
 {
-	if (config_filename) {
-		parse_state = FORCED_CONFIG;
-		parse_include(config_filename, true, early, 0);
-		parse_state = CMDLINE;
+	if (!remove_args) {
+		(*idx)++;
 		return;
 	}
-
-	parse_state = TOPLEVEL_CONFIG;
-	parse_implied_config_file(config_basedir, NULL, early);
-	parse_state = NETWORK_CONFIG;
-	parse_implied_config_file(config_basedir, chainparams->network_name, early);
-	parse_state = CMDLINE;
+	memmove(argv + *idx, argv + 1 + *idx, (*argc - *idx) * sizeof(char *));
+	(*argc)--;
 }

-void initial_config_opts(const tal_t *ctx,
+/* Now all options are known, we can turn cmdline into configvars */
+static struct configvar **gather_cmdline_args(const tal_t *ctx,
+					      int *argc, char **argv,
+					      bool remove_args)
+{
+	struct configvar **cvs = tal_arr(ctx, struct configvar *, 0);
+
+	assert(argv[*argc] == NULL);
+	for (size_t i = 1; argv[i];) {
+		struct opt_table *ot;
+		const char *configline, *arg, *optarg;
+		enum configvar_src src;
+		bool extra_arg;
+
+		/* End of options? */
+		if (streq(argv[i], "--"))
+			break;
+
+		if (!strstarts(argv[i], "-")) {
+			i++;
+			continue;
+		}
+
+		if (strstarts(argv[i], "--")) {
+			arg = argv[i] + 2;
+			ot = opt_find_long(arg, &optarg);
+			src = CONFIGVAR_CMDLINE;
+		} else {
+			/* FIXME: We don't handle multiple short
+			 * options here! */
+			arg = argv[i] + 1;
+			ot = opt_find_short(arg[0]);
+			optarg = NULL;
+			src = CONFIGVAR_CMDLINE_SHORT;
+		}
+		if (ot) {
+			extra_arg = (ot->type & OPT_HASARG) && !optarg;
+		} else {
+			/* Unknown (yet!).  Guess if next arg is for this! */
+			extra_arg = ((src == CONFIGVAR_CMDLINE_SHORT
+				      || !strchr(arg, '='))
+				     && argv[i+1]
+				     && !strstarts(argv[i+1], "-"));
+		}
+		finished_arg(argc, argv, &i, remove_args);
+		/* We turn `--foo bar` into `--foo=bar` here */
+		if (extra_arg) {
+			configline = tal_fmt(tmpctx, "%s=%s", arg, argv[i]);
+			finished_arg(argc, argv, &i, remove_args);
+		} else {
+			configline = arg;
+		}
+		tal_arr_expand(&cvs, configvar_new(cvs, src,
+						   NULL, 0, configline));
+	}
+	assert(argv[*argc] == NULL);
+	return cvs;
+}
+
+void minimal_config_opts(const tal_t *ctx,
 			 int argc, char *argv[],
-			 char **config_filename,
-			 char **config_basedir,
 			 char **config_netdir,
 			 char **rpc_filename)
 {
+	char *unused_filename, *unused_basedir;
+
+	initial_config_opts(tmpctx, &argc, argv, false,
+			    &unused_filename,
+			    &unused_basedir,
+			    config_netdir,
+			    rpc_filename);
+	tal_steal(ctx, *config_netdir);
+	tal_steal(ctx, *rpc_filename);
+}
+
+struct configvar **initial_config_opts(const tal_t *ctx,
+				       int *argc, char *argv[],
+				       bool remove_args,
+				       char **config_filename,
+				       char **config_basedir,
+				       char **config_netdir,
+				       char **rpc_filename)
+{
+	struct configvar **cmdline_cvs, **config_cvs, **cvs;
+
 	options_ctx = ctx;

-	/* First, they could specify a config, which specifies a lightning dir
-	 * or a network. */
+	/* This helps opt_usage. */
+	opt_argv0 = argv[0];
+
+	/* Default chain (a global) is bitcoin. */
+	chainparams = chainparams_for_network("bitcoin");
+
+	/* First, they could specify a config, or base dir. */
 	*config_filename = NULL;
-	opt_register_early_arg("--conf=<file>", opt_set_abspath, NULL,
-			       config_filename,
-			       "Specify configuration file");
-
-	/* Cmdline can also set lightning-dir. */
-	*config_basedir = NULL;
-	opt_register_early_arg("--lightning-dir=<dir>",
-			       opt_set_abspath, NULL,
-			       config_basedir,
-			       "Set base directory: network-specific subdirectory is under here");
-
-	/* Handle --version (and exit) here too */
-	opt_register_version();
-
-	opt_early_parse_incomplete(argc, argv, opt_log_stderr_exit);
-
-	/* Now, reset and ignore --conf option from now on. */
-	opt_free_table();
-
-	/* This is only ever valid on cmdline */
 	opt_register_early_arg("--conf=<file>",
-			       opt_restricted_cmdline,
-			       /* This doesn't show if NULL! */
+			       opt_set_config_filename,
+			       /* Doesn't show if it's NULL! */
 			       opt_show_charp,
 			       config_filename,
 			       "Specify configuration file");
-
-	/* If they set --conf it can still set --lightning-dir */
-	if (!*config_filename) {
-		opt_register_early_arg("--lightning-dir=<dir>",
-				       opt_restricted_forceconf_only, opt_show_charp,
-				       config_basedir,
-				       "Set base directory: network-specific subdirectory is under here");
-	} else {
-		opt_register_early_arg("--lightning-dir=<dir>",
-				       opt_set_abspath, NULL,
-				       config_basedir,
-				       "Set base directory: network-specific subdirectory is under here");
-	}
-
-	/* Now, config file (or cmdline) can set network and lightning-dir */
-
-	/* We need to know network early, so we can set defaults (which normal
-	 * options can change) and default config_netdir */
+	*config_basedir = default_base_configdir(ctx);
+	opt_register_early_arg("--lightning-dir=<dir>",
+			       opt_set_lightning_dir, opt_show_charp,
+			       config_basedir,
+			       "Set base directory: network-specific subdirectory is under here");
 	opt_register_early_arg("--network", opt_set_network, opt_show_network,
 			       NULL,
 			       "Select the network parameters (bitcoin, testnet,"
@@ -355,70 +340,81 @@ void initial_config_opts(const tal_t *ctx,
 	opt_register_early_noarg("--mainnet",
 				 opt_set_specific_network, "bitcoin",
 				 "Alias for --network=bitcoin");
+	/* Handle --version (and exit) here too */
+	opt_register_version();
+
+	/* For convenience, we set deprecated_apis and rpc_filename now, too */
 	opt_register_early_arg("--allow-deprecated-apis",
 			       opt_set_bool_arg, opt_show_bool,
 			       &deprecated_apis,
 			       "Enable deprecated options, JSONRPC commands, fields, etc.");

-	/* Read config file first, since cmdline must override */
-	if (*config_filename)
-		parse_include(*config_filename, true, true, 0);
-	else
-		parse_implied_config_file(*config_basedir, NULL, true);
-	opt_early_parse_incomplete(argc, argv, opt_log_stderr_exit);
+	/* Allow them to override rpc-file too. */
+	*rpc_filename = default_rpcfile(ctx);
+	opt_register_early_arg("--rpc-file", opt_set_talstr, opt_show_charp,
+			       rpc_filename,
+			       "Set JSON-RPC socket (or /dev/tty)");

-	/* We use a global (in common/utils.h) for the chainparams. */
-	if (!chainparams)
-		chainparams = chainparams_for_network("bitcoin");
+	cmdline_cvs = gather_cmdline_args(tmpctx, argc, argv, remove_args);
+	parse_configvars(cmdline_cvs, true, false);

-	if (!*config_basedir)
-		*config_basedir = default_base_configdir(ctx);
+	/* Base default or direct config can set network */
+	if (*config_filename) {
+		config_cvs = gather_file_configvars(NULL,
+						    CONFIGVAR_EXPLICIT_CONF,
+						    *config_filename, true, 0);
+	} else {
+		struct configvar **base_cvs, **net_cvs;
+		char *dir = path_join(tmpctx, take(path_cwd(NULL)), *config_basedir);
+		/* Optional: .lightning/config */
+		base_cvs = gather_file_configvars(tmpctx,
+						  CONFIGVAR_BASE_CONF,
+						  path_join(tmpctx, dir, "config"),
+						  false, 0);
+		/* This might set network! */
+		parse_configvars(configvar_join(tmpctx, base_cvs, cmdline_cvs),
+				 true, false);
+
+		/* Now, we can get network config */
+		dir = path_join(tmpctx, dir, chainparams->network_name);
+		net_cvs = gather_file_configvars(tmpctx,
+						 CONFIGVAR_NETWORK_CONF,
+						 path_join(tmpctx, dir, "config"),
+						 false, 0);
+		config_cvs = configvar_join(NULL, take(base_cvs), take(net_cvs));
+	}
+	cvs = configvar_join(ctx, take(config_cvs), cmdline_cvs);
+
+	/* This will be called again, once caller has added their own
+	 * early vars! */
+	parse_configvars_early(cvs);

 	*config_netdir
 		= path_join(NULL, *config_basedir, chainparams->network_name);

 	/* Make sure it's absolute */
 	*config_netdir = path_join(ctx, take(path_cwd(NULL)), take(*config_netdir));
-
-	/* Now, reset and ignore those options from now on. */
-	opt_free_table();
-
-	opt_register_early_arg("--conf=<file>",
-			       opt_restricted_cmdline,
-			       /* This doesn't show if NULL! */
-			       opt_show_charp,
-			       config_filename,
-			       "Specify configuration file");
-
-	/* This is never in a default config file (since we used the defaults to find it!). */
-	opt_register_early_arg("--lightning-dir=<dir>",
-			       opt_restricted_forceconf_only, opt_show_charp,
-			       config_basedir,
-			       "Set base directory: network-specific subdirectory is under here");
-	opt_register_early_arg("--network",
-			       opt_restricted_toplevel, opt_show_network,
-			       NULL,
-			       "Select the network parameters (bitcoin, testnet,"
-			       " signet, regtest, litecoin or litecoin-testnet)");
-	opt_register_early_noarg("--mainnet",
-				 opt_restricted_toplevel_noarg, NULL,
-				 "Alias for --network=bitcoin");
-	opt_register_early_noarg("--testnet",
-				 opt_restricted_toplevel_noarg, NULL,
-				 "Alias for --network=testnet");
-	opt_register_early_noarg("--signet",
-				 opt_restricted_toplevel_noarg, NULL,
-				 "Alias for --network=signet");
-
-	/* They can set this later, it's just less effective. */
-	opt_register_early_arg("--allow-deprecated-apis",
-			       opt_set_bool_arg, opt_show_bool,
-			       &deprecated_apis,
-			       "Enable deprecated options, JSONRPC commands, fields, etc.");
-
-	/* Set this up for when they parse cmdline proper. */
-	*rpc_filename = default_rpcfile(ctx);
-	opt_register_arg("--rpc-file", opt_set_talstr, opt_show_charp,
-			 rpc_filename,
-			 "Set JSON-RPC socket (or /dev/tty)");
+	return cvs;
+}
+
+void parse_configvars_early(struct configvar **cvs)
+{
+	parse_configvars(cvs, true, false);
+}
+
+void parse_configvars_final(struct configvar **cvs,
+			    bool full_knowledge)
+{
+	parse_configvars(cvs, false, full_knowledge);
+	configvar_finalize_overrides(cvs);
+}
+
+bool is_restricted_ignored(const void *fn)
+{
+	return fn == opt_set_specific_network;
+}
+
+bool is_restricted_print_if_nonnull(const void *fn)
+{
+	return fn == opt_set_config_filename;
 }
--- a/common/configdir.h
+++ b/common/configdir.h
@@ -7,26 +7,37 @@
 * them early. */
 extern bool deprecated_apis;

+/* Unless overridden, we exit with status 1 when option parsing fails */
+extern int opt_exitcode;
+
 /* Helper for options which are tal() strings. */
 char *opt_set_talstr(const char *arg, char **p);

 /* Initial options setup */
 void setup_option_allocators(void);

-/* Parse minimal config options and files */
-void initial_config_opts(const tal_t *ctx,
+/* Minimal config parsing for tools: use opt_early_parse/opt_parse after */
+void minimal_config_opts(const tal_t *ctx,
 			 int argc, char *argv[],
-			 char **config_filename,
-			 char **config_basedir,
 			 char **config_netdir,
 			 char **rpc_filename);

-/* If they specify --conf, we just read that.
- * Otherwise, we read basedir/config (toplevel), and basedir/<network>/config
- * (network-level) */
-void parse_config_files(const char *config_filename,
-			const char *config_basedir,
-			bool early);
+/* Parse initial config options and files */
+struct configvar **initial_config_opts(const tal_t *ctx,
+				       int *argc, char *argv[],
+				       bool remove_args,
+				       char **config_filename,
+				       char **config_basedir,
+				       char **config_netdir,
+				       char **rpc_filename);
+
+/* This is called before we know all the options. */
+void parse_configvars_early(struct configvar **cvs);
+
+/* This is called once, after we know all the options (if full_knowledge
+ * is false, ignore unknown non-cmdline options). */
+void parse_configvars_final(struct configvar **cvs,
+			    bool full_knowledge);

 /* For listconfigs to detect. */
 bool is_restricted_ignored(const void *fn);