common: make sphinx code ignorant of payload format.

Now "raw_payload" is always the complete string (including realm or length
bytes at the front).

This has several effects:
1. We can receive an decrypt an onion which is grossly malformed.
2. We can still hand this to the htlc_accepted hook.
3. We then fail it unless the htlc_accepted accepts it manually.
4. The createonion API now takes the raw payload, and does not know
   anything about "style".

The only caveat is that the sphinx code needs to know the payload
length: we have a call for that, which simply tells it to copy the
entire onion (and treat us as the final node) if it's invalid.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

tests/test_pay.py

@@ -2449,22 +2449,25 @@ def test_createonion_rpc(node_factory):
     l1 = node_factory.get_node()
 
     hops = [{
-        "style": "legacy",
         "pubkey": "02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619",
-        "payload": "0000000000000000000000000000000000000000"
+        # legacy
+        "payload": "000000000000000000000000000000000000000000000000000000000000000000"
     }, {
         "pubkey": "0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c",
-        "payload": "0101010101010101000000000000000100000001"
+        # tlv (20 bytes)
+        "payload": "140101010101010101000000000000000100000001"
     }, {
         "pubkey": "027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007",
-        "payload": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"
+        # TLV (256 bytes)
+        "payload": "fd0100000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"
     }, {
         "pubkey": "032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991",
-        "payload": "0303030303030303000000000000000300000003"
+        # tlv (20 bytes)
+        "payload": "140303030303030303000000000000000300000003"
     }, {
-        "style": "legacy",
         "pubkey": "02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145",
-        "payload": "0404040404040404000000000000000400000004"
+        # legacy
+        "payload": "000404040404040404000000000000000400000004000000000000000000000000"
     }]
 
     res = l1.rpc.createonion(hops=hops, assocdata="BB" * 32)
@@ -2475,6 +2478,7 @@ def test_createonion_rpc(node_factory):
                              session_key="41" * 32)
     # The trailer is generated using the filler and can be ued as a
     # checksum. This trailer is from the test-vector in the specs.
+    print(res)
     assert(res['onion'].endswith('be89e4701eb870f8ed64fafa446c78df3ea'))
 
 
@@ -2491,7 +2495,8 @@ def test_sendonion_rpc(node_factory):
     def serialize_payload(n):
         block, tx, out = n['channel'].split('x')
         payload = hexlify(struct.pack(
-            "!QQL",
+            "!BQQL",
+            0,
             int(block) << 40 | int(tx) << 16 | int(out),
             int(n['amount_msat']),
             blockheight + n['delay'])).decode('ASCII')
@@ -2503,14 +2508,12 @@ def test_sendonion_rpc(node_factory):
     for h, n in zip(route[:-1], route[1:]):
         # We tell the node h about the parameters to use for n (a.k.a. h + 1)
         hops.append({
-            "style": "legacy",
             "pubkey": h['id'],
             "payload": serialize_payload(n)
         })
 
     # The last hop has a special payload:
     hops.append({
-        "style": "legacy",
         "pubkey": route[-1]['id'],
         "payload": serialize_payload(route[-1])
     })