lightningd: chdir as soon as we know lightning dir.

This is easy since we did the option parsing cleanup, but it has the
effect that plugins are launched from the lightning-dir.  Now
we have dynamic plugins, this means startup and post-startup plugins
experience the same environment.

This is absolutely a desirable thing: they can just drop files in
their cwd rather than having to move (including, I might note, core
files!).

We also highlight the change in various places (and a drive-up update
of PLUGINS.md which says you have to use --plugin).

The next patch adds a backwards compatibility wedge for old users of
relative plugin paths.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

CHANGELOG.md

@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - lightningd: check bitcoind version when setup topology and confirm the version not older than v0.15.0.
 - startup: space out reconnections on startup if we have more than 5 peers.
 - JSON API: `listforwards` includes the 'payment_hash' field.
+- plugins: now always run from the `lightning-dir` for easy local storage.
 
 ### Deprecated
 

doc/PLUGINS.md

@@ -27,13 +27,16 @@ executable (e.g. use `chmod a+x plugin_name`)
 During startup of `lightningd` you can use the `--plugin=` option to
 register one or more plugins that should be started. In case you wish
 to start several plugins you have to use the `--plugin=` argument
-once for each plugin. An example call might look like:
+once for each plugin (or `--plugin-dir` or place them in the default
+plugin dirs, usually `/usr/local/libexec/c-lightning/plugins` and 
+`~/.lightningd/plugins`). An example call might look like:
 
 ```
 lightningd --plugin=/path/to/plugin1 --plugin=path/to/plugin2
 ```
 
-`lightningd` will write JSON-RPC requests to the plugin's `stdin` and
+`lightningd` run your plugins from the `--lightning-dir`, then
+will write JSON-RPC requests to the plugin's `stdin` and
 will read replies from its `stdout`. To initialize the plugin two RPC
 methods are required:
 

doc/lightningd-config.5

@@ -2,12 +2,12 @@
 .\"     Title: lightningd-config
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
-.\"      Date: 07/29/2019
+.\"      Date: 08/02/2019
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "LIGHTNINGD\-CONFIG" "5" "07/29/2019" "\ \&" "\ \&"
+.TH "LIGHTNINGD\-CONFIG" "5" "08/02/2019" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -34,7 +34,7 @@ lightningd-config \- Lightning daemon configuration file
 \fB~/\&.lightning/config\fR
 .SH "DESCRIPTION"
 .sp
-When lightningd(8) starts up, it reads a configuration file\&. By default that is \fIconfig\fR in the \fB\&.lightning\fR subdirectory of the home directory (if it exists), but that can be changes by the \fI\-\-lightning\-dir\fR or \fI\-\-conf\fR options on the lightningd(8) command line\&.
+When lightningd(8) starts up, it reads a configuration file\&. By default that is \fIconfig\fR in the \fB\&.lightning\fR subdirectory of the home directory (if it exists), but that can be changed by the \fI\-\-lightning\-dir\fR or \fI\-\-conf\fR options on the lightningd(8) command line\&.
 .sp
 Configuration file options are processed first, then command line options: later options override earlier ones except \fIaddr\fR options which accumulate\&.
 .sp
@@ -142,6 +142,8 @@ Number of blocks to rescan from the current head, or absolute blockheight if neg
 .RS 4
 Sets the working directory\&. All files (except
 \fI\-\-conf\fR
+and
+\fI\-\-lightning\-dir\fR
 on the command line) are relative to this\&.
 .RE
 .PP

doc/lightningd-config.5.txt

@@ -15,7 +15,7 @@ DESCRIPTION
 
 When lightningd(8) starts up, it reads a configuration file.  By
 default that is 'config' in the *.lightning* subdirectory of the home
-directory (if it exists), but that can be changes by the
+directory (if it exists), but that can be changed by the
 '--lightning-dir' or '--conf' options on the lightningd(8) command
 line.
 
@@ -104,8 +104,8 @@ Lightning daemon options
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 *lightning-dir*='DIR'::
-    Sets the working directory. All files (except '--conf' on the command
-    line) are relative to this.
+    Sets the working directory. All files (except '--conf' and '--lightning-dir'
+    on the command line) are relative to this.
 
 *pid-file*='PATH'::
     Specify pid file to write to.

lightningd/options.c

@@ -726,12 +726,19 @@ static char *opt_ignore_talstr(const char *arg, char **p)
 	return NULL;
 }
 
+static char *opt_set_conf(const char *arg, struct lightningd *ld)
+{
+	/* This is a pass-through if arg is absolute */
+	ld->config_filename = path_join(ld, path_cwd(tmpctx), arg);
+	return NULL;
+}
+
 /* Just enough parsing to find config file. */
 static void handle_minimal_config_opts(struct lightningd *ld,
 				       int argc, char *argv[])
 {
-	opt_register_early_arg("--conf=<file>", opt_set_talstr, NULL,
-			       &ld->config_filename,
+	opt_register_early_arg("--conf=<file>", opt_set_conf, NULL,
+			       ld,
 			       "Specify configuration file. Relative paths will be prefixed by lightning-dir location. (default: config)");
 
 	ld->config_dir = default_configdir(ld);
@@ -975,6 +982,19 @@ void handle_early_opts(struct lightningd *ld, int argc, char *argv[])
 	/*~ Handle --conf and --lightning-dir super-early. */
 	handle_minimal_config_opts(ld, argc, argv);
 
+	/*~ Move into config dir: this eases path manipulation and also
+	 * gives plugins a good place to store their stuff. */
+	if (chdir(ld->config_dir) != 0) {
+		log_unusual(ld->log, "Creating configuration directory %s",
+			    ld->config_dir);
+		if (mkdir(ld->config_dir, 0700) != 0)
+			fatal("Could not make directory %s: %s",
+			      ld->config_dir, strerror(errno));
+		if (chdir(ld->config_dir) != 0)
+			fatal("Could not change directory %s: %s",
+			      ld->config_dir, strerror(errno));
+	}
+
 	/*~ The ccan/opt code requires registration then parsing; we
 	 *  mimic this API here, even though they're on separate lines.*/
 	register_opts(ld);
@@ -989,7 +1009,6 @@ void handle_early_opts(struct lightningd *ld, int argc, char *argv[])
 
 	/* Early cmdline options now override config file options. */
 	opt_early_parse_incomplete(argc, argv, opt_log_stderr_exit);
-
 }
 
 void handle_opts(struct lightningd *ld, int argc, char *argv[])
@@ -999,18 +1018,6 @@ void handle_opts(struct lightningd *ld, int argc, char *argv[])
 	 * handle_early_opts */
 	opt_parse_from_config(ld, false);
 
-	/* Move to config dir, to save ourselves the hassle of path manip. */
-	if (chdir(ld->config_dir) != 0) {
-		log_unusual(ld->log, "Creating configuration directory %s",
-			    ld->config_dir);
-		if (mkdir(ld->config_dir, 0700) != 0)
-			fatal("Could not make directory %s: %s",
-			      ld->config_dir, strerror(errno));
-		if (chdir(ld->config_dir) != 0)
-			fatal("Could not change directory %s: %s",
-			      ld->config_dir, strerror(errno));
-	}
-
 	/* Now parse cmdline, which overrides config. */
 	opt_parse(&argc, argv, opt_log_stderr_exit);
 	if (argc != 1)