Improved propagation peer list display

Update Guide.py

README.md

@@ -30,7 +30,36 @@ The easiest way to install Nomad Network is via pip:
 
 ```bash
 # Install Nomad Network and dependencies
-pip3 install nomadnet
+pip install nomadnet
+
+# Run the client
+nomadnet
+
+# Or alternatively run as a daemon, with no user interface
+nomadnet --daemon
+
+# List options
+nomadnet --help
+```
+
+If you are using an operating system that blocks normal user package installation via `pip`, you can return `pip` to normal behaviour by editing the `~/.config/pip/pip.conf` file, and adding the following directive in the `[global]` section:
+
+```text
+[global]
+break-system-packages = true
+```
+
+Alternatively, you can use the `pipx` tool to install Nomad Network in an isolated environment:
+
+```bash
+# Install Nomad Network
+pipx install nomadnet
+
+# Optionally install Reticulum utilities
+pipx install rns
+
+# Optionally install standalone LXMF utilities
+pipx install lxmf
 
 # Run the client
 nomadnet

nomadnet/Directory.py

@@ -34,6 +34,7 @@ class Directory:
     aspect_filter = "nomadnetwork.node"
     @staticmethod
     def received_announce(destination_hash, announced_identity, app_data):
+        try:
             app = nomadnet.NomadNetworkApp.get_shared_instance()
 
             if not destination_hash in app.ignored_list:
@@ -45,6 +46,10 @@ class Directory:
             else:
                 RNS.log("Ignored announce from "+RNS.prettyhexrep(destination_hash), RNS.LOG_DEBUG)
 
+        except Exception as e:
+            RNS.log("Error while evaluating LXMF destination announce, ignoring announce.", RNS.LOG_DEBUG)
+            RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG)
+
 
     def __init__(self, app):
         self.directory_entries = {}
@@ -113,6 +118,19 @@ class Directory:
 
     def lxmf_announce_received(self, source_hash, app_data):
         if app_data != None:
+            if self.app.compact_stream:
+                try:
+                    remove_announces = []
+                    for announce in self.announce_stream:
+                        if announce[1] == source_hash:
+                            remove_announces.append(announce)
+
+                    for a in remove_announces:
+                        self.announce_stream.remove(a)
+                
+                except Exception as e:
+                    RNS.log("An error occurred while compacting the announce stream. The contained exception was:"+str(e), RNS.LOG_ERROR)
+
             timestamp = time.time()
             self.announce_stream.insert(0, (timestamp, source_hash, app_data, "peer"))
             while len(self.announce_stream) > Directory.ANNOUNCE_STREAM_MAXLENGTH:
@@ -123,6 +141,19 @@ class Directory:
 
     def node_announce_received(self, source_hash, app_data, associated_peer):
         if app_data != None:
+            if self.app.compact_stream:
+                try:
+                    remove_announces = []
+                    for announce in self.announce_stream:
+                        if announce[1] == source_hash:
+                            remove_announces.append(announce)
+
+                    for a in remove_announces:
+                        self.announce_stream.remove(a)
+                
+                except Exception as e:
+                    RNS.log("An error occurred while compacting the announce stream. The contained exception was:"+str(e), RNS.LOG_ERROR)
+
             timestamp = time.time()
             self.announce_stream.insert(0, (timestamp, source_hash, app_data, "node"))
             while len(self.announce_stream) > Directory.ANNOUNCE_STREAM_MAXLENGTH:
@@ -150,6 +181,19 @@ class Directory:
                 break
 
         if not found_node:
+            if self.app.compact_stream:
+                try:
+                    remove_announces = []
+                    for announce in self.announce_stream:
+                        if announce[1] == source_hash:
+                            remove_announces.append(announce)
+
+                    for a in remove_announces:
+                        self.announce_stream.remove(a)
+                
+                except Exception as e:
+                    RNS.log("An error occurred while compacting the announce stream. The contained exception was:"+str(e), RNS.LOG_ERROR)
+
             timestamp = time.time()
             self.announce_stream.insert(0, (timestamp, source_hash, app_data, "pn"))
             while len(self.announce_stream) > Directory.ANNOUNCE_STREAM_MAXLENGTH:

nomadnet/Node.py

@@ -97,7 +97,7 @@ class Node:
         for directory in directories:
             self.scan_files(base_path+"/"+directory)
 
-    def serve_page(self, path, data, request_id, remote_identity, requested_at):
+    def serve_page(self, path, data, request_id, link_id, remote_identity, requested_at):
         RNS.log("Page request "+RNS.prettyhexrep(request_id)+" for: "+str(path), RNS.LOG_VERBOSE)
         try:
             self.app.peer_settings["served_page_requests"] += 1
@@ -152,7 +152,20 @@ class Node:
             if request_allowed:
                 RNS.log("Serving page: "+file_path, RNS.LOG_VERBOSE)
                 if os.access(file_path, os.X_OK):
-                    generated = subprocess.run([file_path], stdout=subprocess.PIPE)
+                    env_map = {}
+                    if "PATH" in os.environ:
+                        env_map["PATH"] = os.environ["PATH"]
+                    if link_id != None:
+                        env_map["link_id"] = RNS.hexrep(link_id, delimit=False)
+                    if remote_identity != None:
+                        env_map["remote_identity"] = RNS.hexrep(remote_identity.hash, delimit=False)
+
+                    if data != None and isinstance(data, dict):
+                        for e in data:
+                            if isinstance(e, str) and (e.startswith("field_") or e.startswith("var_")):
+                                env_map[e] = data[e]
+
+                    generated = subprocess.run([file_path], stdout=subprocess.PIPE, env=env_map)
                     return generated.stdout
                 else:
                     fh = open(file_path, "rb")

nomadnet/NomadNetworkApp.py

@@ -111,6 +111,7 @@ class NomadNetworkApp:
         self.pagespath         = self.configdir+"/storage/pages"
         self.filespath         = self.configdir+"/storage/files"
         self.cachepath         = self.configdir+"/storage/cache"
+        self.examplespath      = self.configdir+"/examples"
 
         self.downloads_path    = os.path.expanduser("~/Downloads")
 
@@ -125,6 +126,7 @@ class NomadNetworkApp:
         self.periodic_lxmf_sync = True
         self.lxmf_sync_interval = 360*60
         self.lxmf_sync_limit    = 8
+        self.compact_stream     = False
 
         if not os.path.isdir(self.storagepath):
             os.makedirs(self.storagepath)
@@ -167,6 +169,16 @@ class NomadNetworkApp:
                 RNS.log("Check your configuration file for errors!", RNS.LOG_ERROR)
                 nomadnet.panic()
         else:
+            if not os.path.isdir(self.examplespath):
+                try:
+                    import shutil
+                    examplespath = os.path.join(os.path.dirname(__file__), "examples")
+                    shutil.copytree(examplespath, self.examplespath, ignore=shutil.ignore_patterns("__pycache__"))
+                
+                except Exception as e:
+                    RNS.log("Could not copy examples into the "+self.examplespath+" directory.", RNS.LOG_ERROR)
+                    RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+            
             RNS.log("Could not load config file, creating default configuration file...")
             self.createDefaultConfig()
             self.firstrun = True
@@ -456,9 +468,8 @@ class NomadNetworkApp:
     def autoselect_propagation_node(self):
         selected_node = None
 
-        if "propagation_node" in self.peer_settings:
+        if "propagation_node" in self.peer_settings and self.peer_settings["propagation_node"] != None:
             selected_node = self.peer_settings["propagation_node"]
-        
         else:
             nodes = self.directory.known_nodes()
             trusted_nodes = []
@@ -688,6 +699,10 @@ class NomadNetworkApp:
                     else:
                         self.lxmf_sync_limit = None
 
+                if option == "compact_announce_stream":
+                    value = self.config["client"].as_bool(option)
+                    self.compact_stream = value
+
                 if option == "user_interface":
                     value = value.lower()
                     if value == "none":
@@ -706,7 +721,7 @@ class NomadNetworkApp:
                                 self.config["textui"]["intro_text"] = "Nomad Network"
 
                             if not "editor" in self.config["textui"]:
-                                self.config["textui"]["editor"] = "editor"
+                                self.config["textui"]["editor"] = "nano"
 
                             if not "glyphs" in self.config["textui"]:
                                 self.config["textui"]["glyphs"] = "unicode"
@@ -919,6 +934,12 @@ lxmf_sync_interval = 360
 # the limit, and download everything every time.
 lxmf_sync_limit = 8
 
+# The announce stream will only show one entry
+# per destination or node by default. You can
+# change this to show as many announces as have
+# been received, for every destination.
+compact_announce_stream = yes
+
 [textui]
 
 # Amount of time to show intro screen
@@ -958,10 +979,8 @@ glyphs = unicode
 # application. On by default.
 mouse_enabled = True
 
-# What editor to use for editing text. By
-# default the operating systems "editor"
-# alias will be used.
-editor = editor
+# What editor to use for editing text.
+editor = nano
 
 # If you don't want the Guide section to
 # show up in the menu, you can disable it.

nomadnet/_version.py

@@ -1 +1 @@
-__version__ = "0.3.0"
+__version__ = "0.3.7"

nomadnet/examples/messageboard/README.md

@@ -0,0 +1,18 @@
+# lxmf_messageboard
+Simple message board that can be hosted on a NomadNet node, messages can be posted by 'conversing' with a unique peer, all messages are then forwarded to the message board.
+
+## How Do I Use It?
+A user can submit messages to the message board by initiating a chat with the message board peer, they are assigned a username (based on the first 5 characters of their address) and their messages are added directly to the message board. The message board can be viewed on a page hosted by a NomadNet node.
+
+An example message board can be found on the reticulum testnet hosted on the SolarExpress Node `<d16df67bff870a8eaa2af6957c5a2d7d>` and the message board peer `<ad713cd3fedf36cc190f0cb89c4be1ff>`
+
+## How Does It Work?
+The message board page itself is hosted on a NomadNet node, you can place the message_board.mu into the pages directory. You can then run the message_board.py script which provides the peer that the users can send messages to. The two parts are joined together using umsgpack and a flat file system similar to NomadNet and Reticulum and runs in the background.
+
+## How Do I Set It Up?
+* Turn on node hosting in NomadNet
+* Put the `message_board.mu` file into `pages` directory in the config file for `NomadNet`. Edit the file to customise from the default page.
+* Run the `message_board.py` script (`python3 message_board.py` either in a `screen` or as a system service), this script uses `NomadNet` and `RNS` libraries and has no additional libraries that need to be installed. Take a note of the message boards address, it is printed on starting the board, you can then place this address in `message_board.mu` file to make it easier for users to interact the board.
+
+## Credits
+* This example application was written and contributed by @chengtripp

nomadnet/examples/messageboard/messageboard.mu

@@ -0,0 +1,41 @@
+#!/bin/python3
+import time
+import os
+import RNS.vendor.umsgpack as msgpack
+
+message_board_peer = 'please_replace'
+userdir = os.path.expanduser("~")
+
+if os.path.isdir("/etc/nomadmb") and os.path.isfile("/etc/nomadmb/config"):
+    configdir = "/etc/nomadmb"
+elif os.path.isdir(userdir+"/.config/nomadmb") and os.path.isfile(userdir+"/.config/nomadmb/config"):
+    configdir = userdir+"/.config/nomadmb"
+else:
+    configdir = userdir+"/.nomadmb"
+
+storagepath  = configdir+"/storage"
+if not os.path.isdir(storagepath):
+    os.makedirs(storagepath)
+
+boardpath = configdir+"/storage/board"
+
+print('`!`F222`Bddd`cNomadNet Message Board')
+
+print('-')
+print('`a`b`f')
+print("")
+print("To add a message to the board just converse with the NomadNet Message Board at `[lxmf@{}]".format(message_board_peer))
+time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
+print("Last Updated: {}".format(time_string))
+print("")
+print('>Messages')
+print("  Date       Time    Username     Message")
+f = open(boardpath, "rb")
+board_contents = msgpack.unpack(f)
+board_contents.reverse()
+
+for content in board_contents:
+    print("`a{}".format(content.rstrip()))
+    print("")
+
+f.close()

nomadnet/examples/messageboard/messageboard.py

@@ -0,0 +1,187 @@
+# Simple message board that can be hosted on a NomadNet node, messages can be posted by 'conversing' with a unique peer, all messages are then forwarded to the message board.
+# https://github.com/chengtripp/lxmf_messageboard
+
+import RNS
+import LXMF
+import os, time
+from queue import Queue
+import RNS.vendor.umsgpack as msgpack
+
+display_name = "NomadNet Message Board"
+max_messages = 20
+
+def setup_lxmf():
+    if os.path.isfile(identitypath):
+        identity = RNS.Identity.from_file(identitypath)
+        RNS.log('Loaded identity from file', RNS.LOG_INFO)
+    else:
+        RNS.log('No Primary Identity file found, creating new...', RNS.LOG_INFO)
+        identity = RNS.Identity()
+        identity.to_file(identitypath)
+
+    return identity
+
+def lxmf_delivery(message):
+    # Do something here with a received message
+    RNS.log("A message was received: "+str(message.content.decode('utf-8')))
+
+    message_content = message.content.decode('utf-8')
+    source_hash_text = RNS.hexrep(message.source_hash, delimit=False)
+
+    #Create username (just first 5 char of your addr)
+    username = source_hash_text[0:5]
+
+    RNS.log('Username: {}'.format(username), RNS.LOG_INFO)
+
+    time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(message.timestamp))
+    new_message = '{} {}: {}\n'.format(time_string, username, message_content)
+
+    # Push message to board
+    # First read message board (if it exists
+    if os.path.isfile(boardpath):
+        f = open(boardpath, "rb")
+        message_board = msgpack.unpack(f)
+        f.close()
+    else:
+        message_board = []
+
+    #Check we aren't doubling up (this can sometimes happen if there is an error initially and it then gets fixed)
+    if new_message not in message_board:
+        # Append our new message to the list
+        message_board.append(new_message)
+
+    # Prune the message board if needed
+    while len(message_board) > max_messages:
+        RNS.log('Pruning Message Board')
+        message_board.pop(0)
+
+    # Now open the board and write the updated list
+    f = open(boardpath, "wb")
+    msgpack.pack(message_board, f)
+    f.close()
+
+    # Send reply
+    message_reply = '{}_{}_Your message has been added to the messageboard'.format(source_hash_text, time.time())
+    q.put(message_reply)
+
+def announce_now(lxmf_destination):
+    lxmf_destination.announce()
+
+def send_message(destination_hash, message_content):
+    try:
+      # Make a binary destination hash from a hexadecimal string
+      destination_hash = bytes.fromhex(destination_hash)
+
+    except Exception as e:
+      RNS.log("Invalid destination hash", RNS.LOG_ERROR)
+      return
+
+    # Check that size is correct
+    if not len(destination_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8:
+      RNS.log("Invalid destination hash length", RNS.LOG_ERROR)
+
+    else:
+      # Length of address was correct, let's try to recall the
+      # corresponding Identity
+      destination_identity = RNS.Identity.recall(destination_hash)
+
+      if destination_identity == None:
+        # No path/identity known, we'll have to abort or request one
+        RNS.log("Could not recall an Identity for the requested address. You have probably never received an announce from it. Try requesting a path from the network first. In fact, let's do this now :)", RNS.LOG_ERROR)
+        RNS.Transport.request_path(destination_hash)
+        RNS.log("OK, a path was requested. If the network knows a path, you will receive an announce with the Identity data shortly.", RNS.LOG_INFO)
+
+      else:
+        # We know the identity for the destination hash, let's
+        # reconstruct a destination object.
+        lxmf_destination = RNS.Destination(destination_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, "lxmf", "delivery")
+
+        # Create a new message object
+        lxm = LXMF.LXMessage(lxmf_destination, local_lxmf_destination, message_content, title="Reply", desired_method=LXMF.LXMessage.DIRECT)
+
+        # You can optionally tell LXMF to try to send the message
+        # as a propagated message if a direct link fails
+        lxm.try_propagation_on_fail = True
+
+        # Send it
+        message_router.handle_outbound(lxm)
+
+def announce_check():
+    if os.path.isfile(announcepath):
+        f = open(announcepath, "r")
+        announce = int(f.readline())
+        f.close()
+    else:
+        RNS.log('failed to open announcepath', RNS.LOG_DEBUG)
+        announce = 1
+
+    if announce > int(time.time()):
+        RNS.log('Recent announcement', RNS.LOG_DEBUG)
+    else:
+        f = open(announcepath, "w")
+        next_announce = int(time.time()) + 1800
+        f.write(str(next_announce))
+        f.close()
+        announce_now(local_lxmf_destination)
+        RNS.log('Announcement sent, expr set 1800 seconds', RNS.LOG_INFO)
+
+#Setup Paths and Config Files
+userdir = os.path.expanduser("~")
+
+if os.path.isdir("/etc/nomadmb") and os.path.isfile("/etc/nomadmb/config"):
+    configdir = "/etc/nomadmb"
+elif os.path.isdir(userdir+"/.config/nomadmb") and os.path.isfile(userdir+"/.config/nomadmb/config"):
+    configdir = userdir+"/.config/nomadmb"
+else:
+    configdir = userdir+"/.nomadmb"
+
+storagepath  = configdir+"/storage"
+if not os.path.isdir(storagepath):
+    os.makedirs(storagepath)
+
+identitypath = configdir+"/storage/identity"
+announcepath = configdir+"/storage/announce"
+boardpath = configdir+"/storage/board"
+
+# Message Queue
+q = Queue(maxsize = 5)
+
+# Start Reticulum and print out all the debug messages
+reticulum = RNS.Reticulum(loglevel=RNS.LOG_VERBOSE)
+
+# Create a Identity.
+current_identity = setup_lxmf()
+
+# Init the LXMF router
+message_router = LXMF.LXMRouter(identity = current_identity, storagepath = configdir)
+
+# Register a delivery destination (for yourself)
+# In this example we use the same Identity as we used
+# to instantiate the LXMF router. It could be a different one,
+# but it can also just be the same, depending on what you want.
+local_lxmf_destination = message_router.register_delivery_identity(current_identity, display_name=display_name)
+
+# Set a callback for when a message is received
+message_router.register_delivery_callback(lxmf_delivery)
+
+# Announce node properties
+
+RNS.log('LXMF Router ready to receive on: {}'.format(RNS.prettyhexrep(local_lxmf_destination.hash)), RNS.LOG_INFO)
+announce_check()
+
+while True:
+
+    # Work through internal message queue
+    for i in list(q.queue):
+        message_id = q.get()
+        split_message = message_id.split('_')
+        destination_hash = split_message[0]
+        message = split_message[2]
+        RNS.log('{} {}'.format(destination_hash, message), RNS.LOG_INFO)
+        send_message(destination_hash, message)
+
+    # Check whether we need to make another announcement
+    announce_check()
+
+    #Sleep
+    time.sleep(10)

nomadnet/examples/various/input_fields.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+import os
+env_string = ""
+for e in os.environ:
+  env_string += "{}={}\n".format(e, os.environ[e])
+
+template = """>Fields and Submitting Data
+
+Nomad Network let's you use simple input fields for submitting data to node-side applications. Submitted data, along with other session variables will be available to the node-side script / program as environment variables. This page contains a few examples.
+
+>> Read Environment Variables
+
+{@ENV}
+>>Examples of Fields and Submissions
+
+The following section contains a simple set of fields, and a few different links that submit the field data in different ways.
+
+-=
+
+An input field    : `B444`<username`Entered data>`b
+
+An masked field   : `B444`<!|password`Value of Field>`b
+
+An small field    : `B444`<8|small`test>`b, and some more text.
+
+Two fields        : `B444`<8|one`One>`b `B444`<8|two`Two>`b
+
+The data can be `!`[submitted`:/page/input_fields.mu`username|two]`!.
+
+You can `!`[submit`:/page/input_fields.mu`one|password|small]`! other fields, or just `!`[a single one`:/page/input_fields.mu`username]`!
+
+Or simply `!`[submit them all`:/page/input_fields.mu`*]`!.
+
+Submission links can also `!`[include pre-configured variables`:/page/input_fields.mu`username|two|entitiy_id=4611|action=view]`!.
+
+Or take all fields and `!`[pre-configured variables`:/page/input_fields.mu`*|entitiy_id=4611|action=view]`!.
+
+Or only `!`[pre-configured variables`:/page/input_fields.mu`entitiy_id=4688|task=something]`!
+
+-=
+
+"""
+print(template.replace("{@ENV}", env_string))

nomadnet/ui/TextUI.py

@@ -45,6 +45,8 @@ THEMES = {
             ("list_normal",                 "dark gray", "default",                 "default",                  "#bbb", "default"),
             ("list_untrusted",              "dark red", "default",                  "default",                  "#a22", "default"),
             ("list_focus_untrusted",        "black", "light gray",                  "standout",                 "#810", "#aaa"),
+            ("list_unresponsive",           "yellow", "default",                    "default",                  "#b92", "default"),
+            ("list_focus_unresponsive",     "black", "light gray",                  "standout",                 "#530", "#aaa"),
             ("topic_list_normal",           "light gray", "default",                "default",                  "#ddd", "default"),
             ("browser_controls",            "light gray", "default",                "default",                  "#bbb", "default"),
             ("progress_full",               "black", "light gray",                  "standout",                 "#111", "#bbb"),
@@ -78,6 +80,8 @@ THEMES = {
             ("list_normal",                 "dark gray", "default",                "default",                  "#444", "default"),
             ("list_untrusted",              "dark red", "default",                 "default",                  "#a22", "default"),
             ("list_focus_untrusted",        "black", "dark gray",                  "standout",                 "#810", "#aaa"),
+            ("list_unresponsive",           "yellow", "default",                   "default",                  "#b92", "default"),
+            ("list_focus_unresponsive",     "black", "light gray",                 "standout",                 "#530", "#aaa"),
             ("topic_list_normal",           "dark gray", "default",                "default",                  "#222", "default"),
             ("browser_controls",            "dark gray", "default",                "default",                  "#444", "default"),
             ("progress_full",               "black", "dark gray",                  "standout",                 "#111", "#bbb"),
@@ -217,6 +221,8 @@ class TextUI:
     def unhandled_input(self, key):
         if key == "ctrl q":
             raise urwid.ExitMainLoop
+        elif key == "ctrl e":
+            pass
 
     def display_main(self, loop, user_data):
         self.loop.widget = self.main_display.widget

nomadnet/ui/textui/Browser.py

@@ -23,16 +23,37 @@ class BrowserFrame(urwid.Frame):
             self.delegate.url_dialog()
         elif key == "ctrl s":
             self.delegate.save_node_dialog()
+        elif key == "ctrl b":
+            self.delegate.save_node_dialog()
         elif key == "ctrl g":
             nomadnet.NomadNetworkApp.get_shared_instance().ui.main_display.sub_displays.network_display.toggle_fullscreen()
         elif self.get_focus() == "body":
+            if key == "down" or key == "up":
+                try:
+                    if hasattr(self.delegate, "page_pile") and self.delegate.page_pile:
+                        def df(loop, user_data):
+                            st = None
+                            nf = self.delegate.page_pile.get_focus()
+                            if hasattr(nf, "key_timeout"):
+                                st = nf
+                            elif hasattr(nf, "original_widget"):
+                                no = nf.original_widget
+                                if hasattr(no, "original_widget"):
+                                    st = no.original_widget
+                                else:
+                                    if hasattr(no, "key_timeout"):
+                                        st = no
+                        
+                            if st and hasattr(st, "key_timeout") and hasattr(st, "keypress") and callable(st.keypress):
+                                st.keypress(None, None)
+
+                        nomadnet.NomadNetworkApp.get_shared_instance().ui.loop.set_alarm_in(0.25, df)
+                
+                except Exception as e:
+                    RNS.log("Error while setting up cursor timeout. The contained exception was: "+str(e), RNS.LOG_ERROR)
+                                        
             return super(BrowserFrame, self).keypress(size, key)
-            # if key == "up" and self.delegate.messagelist.top_is_visible:
-            #     nomadnet.NomadNetworkApp.get_shared_instance().ui.main_display.frame.set_focus("header")
-            # elif key == "down" and self.delegate.messagelist.bottom_is_visible:
-            #     self.set_focus("footer")
-            # else:
-            #     return super(ConversationFrame, self).keypress(size, key)
+
         else:
             return super(BrowserFrame, self).keypress(size, key)
 
@@ -62,6 +83,7 @@ class Browser:
         self.aspects = aspects
         self.destination_hash = destination_hash
         self.path = path
+        self.request_data = None
         self.timeout = Browser.DEFAULT_TIMEOUT
         self.last_keypress = None
 
@@ -80,6 +102,7 @@ class Browser:
         self.link_target = None
         self.frame = None
         self.attr_maps = []
+        self.page_pile = None
         self.build_display()
 
         self.history = []
@@ -141,7 +164,42 @@ class Browser:
         else:
             return destination_type
 
-    def handle_link(self, link_target):
+    def handle_link(self, link_target, link_data = None):
+        request_data = None
+        if link_data != None:
+            link_fields = []
+            request_data = {}
+            all_fields = True if "*" in link_data else False
+
+            for e in link_data:
+                if "=" in e:
+                    c = e.split("=")
+                    if len(c) == 2:
+                        request_data["var_"+str(c[0])] = str(c[1])
+                else:
+                    link_fields.append(e)
+
+            def recurse_down(w):
+                target = None
+                if isinstance(w, list):
+                    for t in w:
+                        recurse_down(t)
+                elif isinstance(w, tuple):
+                    for t in w:
+                        recurse_down(t)
+                elif hasattr(w, "contents"):
+                    recurse_down(w.contents)
+                elif hasattr(w, "original_widget"):
+                    recurse_down(w.original_widget)
+                elif hasattr(w, "_original_widget"):
+                    recurse_down(w._original_widget)
+                else:
+                    if hasattr(w, "field_name") and (all_fields or w.field_name in link_fields):
+                        request_data["field_"+w.field_name] = w.get_edit_text()
+                
+            recurse_down(self.attr_maps)
+            RNS.log("Including request data: "+str(request_data), RNS.LOG_DEBUG)
+
         components = link_target.split("@")
         destination_type = None
 
@@ -155,8 +213,9 @@ class Browser:
         if destination_type == "nomadnetwork.node":
             if self.status >= Browser.DISCONECTED:
                 RNS.log("Browser handling link to: "+str(link_target), RNS.LOG_DEBUG)
+                self.browser_footer = urwid.Text("Opening link to: "+str(link_target))
                 try:
-                    self.retrieve_url(link_target)
+                    self.retrieve_url(link_target, request_data)
                 except Exception as e:
                     self.browser_footer = urwid.Text("Could not open link: "+str(e))
                     self.frame.contents["footer"] = (self.browser_footer, self.frame.options())
@@ -219,6 +278,7 @@ class Browser:
         self.browser_header = urwid.Text("")
         self.browser_footer = urwid.Text("")
 
+        self.page_pile = None
         self.browser_body = urwid.Filler(urwid.Text("Disconnected\n"+self.g["arrow_l"]+"  "+self.g["arrow_r"], align="center"), "middle")
 
         self.frame = BrowserFrame(self.browser_body, header=self.browser_header, footer=self.browser_footer)
@@ -266,6 +326,7 @@ class Browser:
     def update_display(self):
         if self.status == Browser.DISCONECTED:
             self.display_widget.set_attr_map({None: "inactive_text"})
+            self.page_pile = None
             self.browser_body = urwid.Filler(urwid.Text("Disconnected\n"+self.g["arrow_l"]+"  "+self.g["arrow_r"], align="center"), "middle")
             self.browser_footer = urwid.Text("")
             self.browser_header = urwid.Text("")
@@ -314,6 +375,8 @@ class Browser:
 
     def update_page_display(self):
         pile = urwid.Pile(self.attr_maps)
+        pile.automove_cursor_on_scroll = True
+        self.page_pile = pile
         self.browser_body = urwid.AttrMap(ScrollBar(Scrollable(pile, force_forward_keypress=True), thumb_char="\u2503", trough_char=" "), "scrollbar")
 
     def identify(self):
@@ -326,6 +389,7 @@ class Browser:
         if self.link != None:
             self.link.teardown()
         
+        self.request_data = None
         self.attr_maps = []
         self.status = Browser.DISCONECTED
         self.response_progress = 0
@@ -340,7 +404,7 @@ class Browser:
         self.update_display()
 
 
-    def retrieve_url(self, url):
+    def retrieve_url(self, url, request_data = None):
         self.previous_destination_hash = self.destination_hash
         self.previous_path = self.path
 
@@ -393,6 +457,7 @@ class Browser:
             else:
                 self.set_destination_hash(destination_hash)
                 self.set_path(path)
+                self.set_request_data(request_data)
                 self.load_page()
 
     def set_destination_hash(self, destination_hash):
@@ -406,6 +471,8 @@ class Browser:
     def set_path(self, path):
         self.path = path
 
+    def set_request_data(self, request_data):
+        self.request_data = request_data
 
     def set_timeout(self, timeout):
         self.timeout = timeout
@@ -612,7 +679,11 @@ class Browser:
 
 
     def load_page(self):
+        if self.request_data == None:
             cached = self.get_cached(self.current_url())
+        else:
+            cached = None
+
         if cached:
             self.status = Browser.DONE
             self.page_data = cached
@@ -646,7 +717,15 @@ class Browser:
                 page_data = b"The requested local page did not exist in the file system"
                 if os.path.isfile(page_path):
                     if os.access(page_path, os.X_OK):
-                        generated = subprocess.run([page_path], stdout=subprocess.PIPE)
+                        if self.request_data != None:
+                            env_map = self.request_data
+                        else:
+                            env_map = {}
+                            
+                        if "PATH" in os.environ:
+                            env_map["PATH"] = os.environ["PATH"]
+
+                        generated = subprocess.run([page_path], stdout=subprocess.PIPE, env=env_map)
                         page_data = generated.stdout
                     else:
                         file = open(page_path, "rb")
@@ -677,7 +756,7 @@ class Browser:
     def __load(self):
         # If an established link exists, but it doesn't match the target
         # destination, we close and clear it.
-        if self.link != None and self.link.destination.hash != self.destination_hash:
+        if self.link != None and (self.link.destination.hash != self.destination_hash or self.link.status != RNS.Link.ACTIVE):
             self.link.teardown()
             self.link = None
 
@@ -733,7 +812,7 @@ class Browser:
         self.update_display()
         receipt = self.link.request(
             self.path,
-            data = None,
+            data = self.request_data,
             response_callback = self.response_received,
             failed_callback = self.request_failed,
             progress_callback = self.response_progressed
@@ -926,6 +1005,11 @@ class Browser:
                 self.response_transfer_size = None
 
                 self.update_display()
+                if self.link != None:
+                    try:
+                        self.link.teardown()
+                    except Exception as e:
+                        pass
         else:
             self.status = Browser.REQUEST_FAILED
             self.response_progress = 0
@@ -933,6 +1017,11 @@ class Browser:
             self.response_transfer_size = None
 
             self.update_display()
+            if self.link != None:
+                try:
+                    self.link.teardown()
+                except Exception as e:
+                    pass
 
 
     def request_timeout(self, request_receipt=None):
@@ -942,6 +1031,11 @@ class Browser:
         self.response_transfer_size = None
 
         self.update_display()
+        if self.link != None:
+            try:
+                self.link.teardown()
+            except Exception as e:
+                pass
 
 
     def response_progressed(self, request_receipt):

nomadnet/ui/textui/Conversations.py

@@ -110,7 +110,10 @@ class ConversationsDisplay():
 
     def delete_selected_conversation(self):
         self.dialog_open = True
-        source_hash = self.ilb.get_selected_item().source_hash
+        item = self.ilb.get_selected_item()
+        if item == None:
+            return
+        source_hash = item.source_hash
 
         def dismiss_dialog(sender):
             self.update_conversation_list()
@@ -140,7 +143,10 @@ class ConversationsDisplay():
     def edit_selected_in_directory(self):
         g = self.app.ui.glyphs
         self.dialog_open = True
-        source_hash_text = self.ilb.get_selected_item().source_hash
+        item = self.ilb.get_selected_item()
+        if item == None:
+            return
+        source_hash_text = item.source_hash
         display_name = self.ilb.get_selected_item().display_name
         if display_name == None:
             display_name = ""

nomadnet/ui/textui/Guide.py

@@ -312,27 +312,38 @@ The distributed message store is resilient to intermittency, and will remain fun
 
 >>Pages
 
-Nomad Network nodes can host pages similar to web pages, that other peers can read and interact with. Pages are written in a compact markup language called `*micron`*. To learn how to write formatted pages with micron, see the `*Markup`* section of this guide (which is, itself, written in micron). Pages can be linked arbitrarily with hyperlinks, that can also link to pages (or other resources) on other nodes.
+Nomad Network nodes can host pages similar to web pages, that other peers can read and interact with. Pages are written in a compact markup language called `*micron`*. To learn how to write formatted pages with micron, see the `*Markup`* section of this guide (which is, itself, written in micron). Pages can be linked together with hyperlinks, that can also link to pages (or other resources) on other nodes.
 
 To add pages to your node, place micron files in the `*pages`* directory of your Nomad Network programs `*storage`* directory. By default, the path to this will be `!~/.nomadnetwork/storage/pages`!. You should probably create the file `!index.mu`! first, as this is the page that will get served by default to a connecting peer.
 
 You can control how long a peer will cache your pages by including the cache header in a page. To do so, the first line of your page must start with `!#!c=X`!, where `!X`! is the cache time in seconds. To tell the peer to always load the page from your node, and never cache it, set the cache time to zero. You should only do this if there is a real need, for example if your page displays dynamic content that `*must`* be updated at every page view. The default caching time is 12 hours. In most cases, you should not need to include the cache control header in your pages.
 
+>> Dynamic Pages
+
 You can use a preprocessor such as PHP, bash, Python (or whatever you prefer) to generate dynamic pages. To do so, just set executable permissions on the relevant page file, and be sure to include the interpreter at the beginning of the file, for example `!#!/usr/bin/python3`!.
 
+Data from fields and link variables will be passed to these scipts or programs as environment variables, and can simply be read by any method for acessing such.
+
+In the `!examples`! directory, you can find various small examples for the use of this feature. The currently included examples are:
+
+ - A messageboard that receives messages over LXMF, contributed by trippcheng
+ - A simple demonstration on how to create fields and read entered data in node-side scripts
+
+By default, you can find the examples in `!~/.nomadnetwork/examples`!. If you build something neat, that you feel would fit here, you are more than welcome to contribute it.
+
 >>Authenticating Users
 
 Sometimes, you don't want everyone to be able to view certain pages or execute certain scripts. In such cases, you can use `*authentication`* to control who gets to run certain requests.
 
-To enable authentication for any page, simply add a new file to your pages directory with ".allowed" added to the file-name of the page. If your page is named "secret_page.mu", just add a file named "secret_page.mu.allowed".
+To enable authentication for any page, simply add a new file to your pages directory with ".allowed" added to the file-name of the page. If your page is named "secret_page.mu", just add a file named "secret_page.allowed".
 
 For each user allowed to access the page, add a line to this file, containing the hash of that users primary identity. Users can find their own identity hash in the `![ Network ]`! part of the program, under `!Local Peer Info`!. If you want to allow access for three different users, your file would look like this:
 
 `Faaa
 `=
-24c2dc2248953e0a3c21
-564a476410f4fca2ae93
-8bb864743f12f53395a1
+d454bcdac0e64fb68ba8e267543ae110
+2b9ff3fb5902c9ca5ff97bdfb239ef50
+7106d5abbc7208bfb171f2dd84b36490
 `=
 ``
 
@@ -348,7 +359,7 @@ Like pages, you can place files you want to make available in the `!~/.nomadnetw
 
 Links to pages and resources in Nomad Network use a simple URL format. Here is an example:
 
-`!1385edace36466a6b3dd:/page/index.mu`!
+`!18176ffddcc8cce1ddf8e3f72068f4a6:/page/index.mu`!
 
 The first part is the 10 byte destination address of the node (represented as readable hexadecimal), followed by the `!:`! character. Everything after the `!:`! represents the request path.
 
@@ -495,6 +506,12 @@ The number of minutes between each automatic sync. The default is equal to 6 hou
 On low-bandwidth networks, it can be useful to limit the amount of messages downloaded in each sync. The default is 8. Set to 0 to download all available messages every time a sync occurs.
 <
 
+>>>
+`!compact_announce_stream = yes`!
+>>>>
+With this option enabled, Nomad Network will only display one entry in the announce stream per destination. Older announces are culled when a new one arrives.
+<
+
 >> Text UI Section
 
 This section hold configuration directives related to the look and feel of the text-based user interface of the program. It is delimited by the `![textui]`! header in the configuration file. Available directives, along with their default values, are as follows:
@@ -994,6 +1011,94 @@ Here is `F00f`_`[a more visible link`72914442a3689add83a09a767963f57c:/page/inde
 
 When links like these are displayed in the built-in browser, clicking on them or activating them using the keyboard will cause the browser to load the specified URL.
 
+>Fields & Requests
+
+Nomad Network let's you use simple input fields for submitting data to node-side applications. Submitted data, along with other session variables will be available to the node-side script / program as environment variables.
+
+>>Request Links
+
+Links can contain request variables and a list of fields to submit to the node-side application. You can include all fields on the page, only specific ones, and any number of request variables. To simply submit all fields on a page to a specified node-side page, create a link like this:
+
+`Faaa
+`=
+`[Submit Fields`:/page/fields.mu`*]
+`=
+``
+
+Note the `!*`! following the extra `!\``! at the end of the path. This `!*`! denotes `*all fields`*. You can also specify a list of fields to include:
+
+`Faaa
+`=
+`[Submit Fields`:/page/fields.mu`username|auth_token]
+`=
+``
+
+If you want to include pre-set variables, you can do it like this:
+
+`Faaa
+`=
+`[Query the System`:/page/fields.mu`username|auth_token|action=view|amount=64]
+`=
+``
+
+>> Fields
+
+Here's an example of creating a field. We'll create a field named `!user_input`! and fill it with the text `!Pre-defined data`!. Note that we are using background color tags to make the field more visible to the user:
+
+`Faaa
+`=
+A simple input field: `B444`<user_input`Pre-defined data>`b
+`=
+``
+
+You must always set a field `*name`*, but you can of course omit the pre-defined value of the field:
+
+`Faaa
+`=
+An empty input field: `B444`<demo_empty`>`b
+`=
+``
+
+You can set the size of the field like this:
+
+`Faaa
+`=
+A sized input field:  `B444`<16|with_size`>`b
+`=
+``
+
+It is possible to mask fields, for example for use with passwords and similar:
+
+`Faaa
+`=
+A masked input field: `B444`<!|masked_demo`hidden text>`b
+`=
+``
+
+And you can of course control all parameters at the same time:
+
+`Faaa
+`=
+Full control: `B444`<!32|all_options`hidden text>`b
+`=
+``
+
+Collecting the above markup produces the following output:
+
+`Faaa`B333
+
+A simple input field: `B444`<user_input`Pre-defined data>`B333
+
+An empty input field: `B444`<demo_empty`>`B333
+
+A sized input field:  `B444`<16|with_size`>`B333
+
+A masked input field: `B444`<!|masked_demo`hidden text>`B333
+
+Full control: `B444`<!32|all_options`hidden text>`B333
+
+``
+
 >Comments
 
 You can insert comments that will not be displayed in the output by starting a line with the # character.
@@ -1026,7 +1131,7 @@ To display literal content, for example source-code, or blocks of text that shou
 `=
 '''
 TOPIC_MARKUP += TOPIC_MARKUP.replace("`=", "\\`=") + "[ micron source for document goes here, we don't want infinite recursion now, do we? ]\n\\`="
-TOPIC_MARKUP += "\n`=\n\n>Closing Remarks\n\nIf you made it all the way here, you should be well equipped to write documents and pages using micron. Thank you for staying with me.\n\n`c\U0001F332\n"
+TOPIC_MARKUP += "\n`=\n\n>Closing Remarks\n\nIf you made it all the way here, you should be well equipped to write documents, pages and applications using micron and Nomad Network. Thank you for staying with me.\n\n`c\U0001F332\n"
 
 
 TOPICS = {

nomadnet/ui/textui/Main.py

@@ -140,6 +140,15 @@ class MainDisplay():
         self.menu_display.start()
 
     def quit(self, sender=None):
+        logterm_pid = None
+        if True or RNS.vendor.platformutils.is_android():
+            if self.sub_displays.log_display != None and self.sub_displays.log_display.log_term != None:
+                if self.sub_displays.log_display.log_term.log_term != None:
+                    logterm_pid = self.sub_displays.log_display.log_term.log_term.pid
+                    if logterm_pid != None:
+                        import os, signal
+                        os.kill(logterm_pid, signal.SIGKILL)
+
         raise urwid.ExitMainLoop
 
 

nomadnet/ui/textui/MicronParser.py

@@ -1,5 +1,6 @@
 import nomadnet
 import urwid
+import random
 import time
 from urwid.util import is_mouse_press
 from urwid.text_layout import calc_coords
@@ -62,11 +63,12 @@ def markup_to_attrmaps(markup, url_delegate = None):
 
     for line in lines:
         if len(line) > 0:
-            display_widget = parse_line(line, state, url_delegate)
+            display_widgets = parse_line(line, state, url_delegate)
         else:
-            display_widget = urwid.Text("")
+            display_widgets = [urwid.Text("")]
         
-        if display_widget != None:
+        if display_widgets != None and len(display_widgets) != 0:
+            for display_widget in display_widgets:
                 attrmap = urwid.AttrMap(display_widget, make_style(state))
                 attrmaps.append(attrmap)
 
@@ -124,7 +126,7 @@ def parse_line(line, state, url_delegate):
 
                         heading_style = first_style
                         output.insert(0, " "*left_indent(state))
-                        return urwid.AttrMap(urwid.Text(output, align=state["align"]), heading_style)
+                        return [urwid.AttrMap(urwid.Text(output, align=state["align"]), heading_style)]
                     else:
                         return None
                 else:
@@ -137,22 +139,56 @@ def parse_line(line, state, url_delegate):
                 else:
                     divider_char = "\u2500"
                 if state["depth"] == 0:
-                    return urwid.Divider(divider_char)
+                    return [urwid.Divider(divider_char)]
                 else:
-                    return urwid.Padding(urwid.Divider(divider_char), left=left_indent(state), right=right_indent(state))
+                    return [urwid.Padding(urwid.Divider(divider_char), left=left_indent(state), right=right_indent(state))]
 
         output = make_output(state, line, url_delegate)
 
         if output != None:
+            text_only = True
+            for o in output:
+                if not isinstance(o, tuple):
+                    text_only = False
+                    break
+
+            if not text_only:
+                widgets = []
+                for o in output:
+                    if isinstance(o, tuple):
+                        if url_delegate != None:
+                            tw = LinkableText(o, align=state["align"], delegate=url_delegate)
+                            tw.in_columns = True
+                        else:
+                            tw = urwid.Text(o, align=state["align"])
+                        
+                        widgets.append(("pack", tw))
+                    else:
+                        if o["type"] == "field":
+                            fw = o["width"]
+                            fd = o["data"]
+                            fn = o["name"]
+                            fs = o["style"]
+                            fmask = "*" if o["masked"] else None
+                            f = urwid.Edit(caption="", edit_text=fd, align=state["align"], multiline=True, mask=fmask)
+                            f.field_name = fn
+                            fa = urwid.AttrMap(f, fs)
+                            widgets.append((fw, fa))
+
+                columns_widget = urwid.Columns(widgets, dividechars=0)
+                text_widget = columns_widget
+                # text_widget = urwid.Text("<"+output+">", align=state["align"])
+
+            else:
                 if url_delegate != None:
                     text_widget = LinkableText(output, align=state["align"], delegate=url_delegate)
                 else:
                     text_widget = urwid.Text(output, align=state["align"])
 
             if state["depth"] == 0:
-                return text_widget
+                return [text_widget]
             else:
-                return urwid.Padding(text_widget, left=left_indent(state), right=right_indent(state))
+                return [urwid.Padding(text_widget, left=left_indent(state), right=right_indent(state))]
         else:
             return None
     else:
@@ -423,6 +459,54 @@ def make_output(state, line, url_delegate):
                     elif c == "a":
                         state["align"] = state["default_align"]
 
+                    elif c == "<":
+                        try:
+                            field_name = None
+                            field_name_end = line[i:].find("`")
+                            if field_name_end == -1:
+                                pass
+                            else:
+                                field_name = line[i+1:i+field_name_end]
+                                field_name_skip = len(field_name)
+                                field_masked = False
+                                field_width = 24
+
+                                if "|" in field_name:
+                                    f_components = field_name.split("|")
+                                    field_flags = f_components[0]
+                                    field_name = f_components[1]
+                                    if "!" in field_flags:
+                                        field_flags = field_flags.replace("!", "")
+                                        field_masked = True
+                                    if len(field_flags) > 0:
+                                        field_width = min(int(field_flags), 256)
+
+                                def sr():
+                                    return "@{"+str(random.randint(1000,9999))+"}"
+                                rsg = sr()
+                                while rsg in line[i+field_name_end:]:
+                                    rsg = sr()
+                                lr = line[i+field_name_end:].replace("\\>", rsg)
+                                endpos = lr.find(">")
+                                
+                                if endpos == -1:
+                                    pass
+                                
+                                else:
+                                    field_data = lr[1:endpos].replace(rsg, "\\>")
+                                    skip = len(field_data)+field_name_skip+2
+                                    field_data = field_data.replace("\\>", ">")
+                                    output.append({
+                                        "type":"field",
+                                        "name": field_name,
+                                        "width": field_width,
+                                        "masked": field_masked,
+                                        "data": field_data,
+                                        "style": make_style(state)
+                                    })
+                        except Exception as e:
+                            pass
+    
                     elif c == "[":
                         endpos = line[i:].find("]")
                         if endpos == -1:
@@ -434,13 +518,20 @@ def make_output(state, line, url_delegate):
                             link_components = link_data.split("`")
                             if len(link_components) == 1:
                                 link_label = ""
+                                link_fields = ""
                                 link_url = link_data
                             elif len(link_components) == 2:
                                 link_label = link_components[0]
                                 link_url = link_components[1]
+                                link_fields = ""
+                            elif len(link_components) == 3:
+                                link_label = link_components[0]
+                                link_url = link_components[1]
+                                link_fields = link_components[2]
                             else:
                                 link_url = ""
                                 link_label = ""
+                                link_fields = ""
 
                             if len(link_url) != 0:
                                 if link_label == "":
@@ -468,6 +559,11 @@ def make_output(state, line, url_delegate):
 
                                 if url_delegate != None:
                                     linkspec = LinkSpec(link_url, orig_spec)
+                                    if link_fields != "":
+                                        lf = link_fields.split("|")
+                                        if len(lf) > 0:
+                                            linkspec.link_fields = lf
+
                                     output.append((linkspec, link_label))
                                 else:
                                     output.append(make_part(state, link_label)) 
@@ -510,6 +606,7 @@ def make_output(state, line, url_delegate):
 class LinkSpec(urwid.AttrSpec):
     def __init__(self, link_target, orig_spec):
         self.link_target = link_target
+        self.link_fields = None
 
         urwid.AttrSpec.__init__(self, orig_spec.foreground, orig_spec.background)
 
@@ -525,12 +622,13 @@ class LinkableText(urwid.Text):
         self.delegate = delegate
         self._cursor_position = 0
         self.key_timeout = 3
+        self.in_columns = False
         if self.delegate != None:
             self.delegate.last_keypress = 0
 
-    def handle_link(self, link_target):
+    def handle_link(self, link_target, link_fields):
         if self.delegate != None:
-            self.delegate.handle_link(link_target)
+            self.delegate.handle_link(link_target, link_fields)
 
     def find_next_part_pos(self, pos, part_positions):
         for position in part_positions:
@@ -588,7 +686,7 @@ class LinkableText(urwid.Text):
             item = self.find_item_at_pos(self._cursor_position)
             if item != None:
                 if isinstance(item, LinkSpec):
-                    self.handle_link(item.link_target)
+                    self.handle_link(item.link_target, item.link_fields)
 
         elif key == "up":
             self._cursor_position = 0
@@ -601,7 +699,11 @@ class LinkableText(urwid.Text):
         elif key == "right":
             old = self._cursor_position
             self._cursor_position = self.find_next_part_pos(self._cursor_position, part_positions)
+
             if self._cursor_position == old:
+                if self.in_columns:
+                    return "right"
+                else:
                     self._cursor_position = 0
                     return "down"
 
@@ -609,6 +711,9 @@ class LinkableText(urwid.Text):
         
         elif key == "left":
             if self._cursor_position > 0:
+                if self.in_columns:
+                    return "left"
+                else:
                     self._cursor_position = self.find_prev_part_pos(self._cursor_position, part_positions)
                     self._invalidate()
 
@@ -646,6 +751,7 @@ class LinkableText(urwid.Text):
             return x, y
 
     def mouse_event(self, size, event, button, x, y, focus):
+        try:
             if button != 1 or not is_mouse_press(event):
                 return False
             else:
@@ -678,9 +784,12 @@ class LinkableText(urwid.Text):
 
                 if item != None:
                     if isinstance(item, LinkSpec):
-                    self.handle_link(item.link_target)
+                        self.handle_link(item.link_target, item.link_fields)
 
                 self._invalidate()
                 self._emit("change")
                 
                 return True
+            
+        except Exception as e:
+            return False

nomadnet/ui/textui/Network.py

@@ -13,9 +13,7 @@ class NetworkDisplayShortcuts():
         self.app = app
         g = app.ui.glyphs
 
-        self.widget = urwid.AttrMap(urwid.Text("[C-l] Nodes/Announces  [C-x] Remove  [C-w] Disconnect  [C-d] Back  [C-f] Forward  [C-r] Reload  [C-u] URL  [C-g] Fullscreen"), "shortcutbar")
-        #   "[C-"+g["arrow_u"]+g["arrow_d"]+"] Navigate Lists"
-
+        self.widget = urwid.AttrMap(urwid.Text("[C-l] Nodes/Announces  [C-x] Remove  [C-w] Disconnect  [C-d] Back  [C-f] Forward  [C-r] Reload  [C-u] URL  [C-g] Fullscreen  [C-s / C-b] Save Node"), "shortcutbar")
 
 class DialogLineBox(urwid.LineBox):
     def keypress(self, size, key):
@@ -239,7 +237,7 @@ class AnnounceInfo(urwid.WidgetWrap):
 
 
 class AnnounceStreamEntry(urwid.WidgetWrap):
-    def __init__(self, app, announce):
+    def __init__(self, app, announce, delegate):
         full_time_format = "%Y-%m-%d %H:%M:%S"
         date_time_format = "%Y-%m-%d"
         time_time_format = "%H:%M:%S"
@@ -250,6 +248,7 @@ class AnnounceStreamEntry(urwid.WidgetWrap):
         source_hash = announce[1]
         announce_type = announce[3]
         self.app = app
+        self.delegate = delegate
         self.timestamp = timestamp
         time_format = app.time_format
         dt = datetime.fromtimestamp(self.timestamp)
@@ -299,11 +298,47 @@ class AnnounceStreamEntry(urwid.WidgetWrap):
         urwid.WidgetWrap.__init__(self, self.display_widget)
 
     def display_announce(self, event, announce):
+      try:
         parent = self.app.ui.main_display.sub_displays.network_display
         info_widget = AnnounceInfo(announce, parent, self.app)
         options = parent.left_pile.options(height_type="weight", height_amount=1)
         parent.left_pile.contents[0] = (info_widget, options)
       
+      except KeyError as e:
+        def dismiss_dialog(sender):
+            self.delegate.parent.close_list_dialogs()
+
+        def confirmed(sender):
+            def close_req(sender):
+                self.delegate.parent.close_list_dialogs()
+
+            dialog_pile.contents[0] = (urwid.Text("\nKeys requested from network\n", align="center"), options)
+            RNS.Transport.request_path(announce[1])
+
+        confirmed_button = urwid.Button("Request keys", on_press=confirmed)
+
+        dialog_pile = urwid.Pile([
+            urwid.Text("The keys for the announced destination could not be recalled. You can wait for an announce to arrive, or request the keys from the network.\n", align="center"),
+            urwid.Columns([
+                ("weight", 0.45, confirmed_button),
+                ("weight", 0.1, urwid.Text("")),
+                ("weight", 0.45, urwid.Button("Close", on_press=dismiss_dialog)),
+            ])
+        ])
+
+        dialog = ListDialogLineBox(
+            dialog_pile,
+            title="Keys Unknown"
+        )
+        confirmed_button.dialog_pile = dialog_pile
+        dialog.delegate = self.delegate.parent
+        bottom = self.delegate
+
+        overlay = urwid.Overlay(dialog, bottom, align="center", width=("relative", 100), valign="middle", height="pack", left=2, right=2)
+
+        options = self.delegate.parent.left_pile.options("weight", 1)
+        self.delegate.parent.left_pile.contents[0] = (overlay, options)
+
     def timestamp(self):
         return self.timestamp
 
@@ -358,7 +393,7 @@ class AnnounceStream(urwid.WidgetWrap):
                 new_entries.insert(0, e)
 
         for e in new_entries:
-            nw = AnnounceStreamEntry(self.app, e)
+            nw = AnnounceStreamEntry(self.app, e, self)
             nw.timestamp = e[0]
             self.widget_list.insert(0, nw)
 
@@ -573,6 +608,10 @@ class KnownNodeInfo(urwid.WidgetWrap):
             node_entry = DirectoryEntry(source_hash, display_name=display_str, trust_level=trust_level, hosts_node=True, identify_on_connect=connect_identify_checkbox.get_state())
             self.app.directory.remember(node_entry)
             self.app.ui.main_display.sub_displays.network_display.directory_change_callback()
+
+            if trust_level == DirectoryEntry.TRUSTED:
+                self.app.autoselect_propagation_node()
+
             show_known_nodes(None)
 
         back_button = ("weight", 0.2, urwid.Button("Back", on_press=show_known_nodes))
@@ -667,7 +706,7 @@ class KnownNodes(urwid.WidgetWrap):
         else:
             self.no_content = True
             widget_style = "inactive_text"
-            self.pile = urwid.Pile([urwid.Text(("warning_text", g["info"]+"\n"), align="center"), SelectText(("warning_text", "Currently, no nodes are known\n\n"), align="center")])
+            self.pile = urwid.Pile([urwid.Text(("warning_text", g["info"]+"\n"), align="center"), SelectText(("warning_text", "Currently, no nodes are saved\n\nCtrl+L to view the announce stream\n\n"), align="center")])
             self.display_widget = urwid.Filler(self.pile, valign="top", height="pack")
 
         urwid.WidgetWrap.__init__(self, urwid.AttrMap(urwid.LineBox(self.display_widget, title="Saved Nodes"), widget_style))
@@ -1498,10 +1537,6 @@ class NetworkDisplay():
         self.local_peer_display.start()
         self.node_info_display.start()
         self.network_stats_display.start()
-        # There seems to be an intermittent memory leak somewhere
-        # in the periodic updating here. The periodic updater should
-        # not be needed anymore, so dis
-        #self.announce_stream_display.start()
 
     def shortcuts(self):
         return self.shortcuts_display
@@ -1541,7 +1576,11 @@ class LXMFPeers(urwid.WidgetWrap):
             self.pile = urwid.Pile([urwid.Text(("warning_text", g["info"]+"\n"), align="center"), SelectText(("warning_text", "Currently, no LXMF nodes are peered\n\n"), align="center")])
             self.display_widget = urwid.Filler(self.pile, valign="top", height="pack")
 
-        urwid.WidgetWrap.__init__(self, urwid.AttrMap(urwid.LineBox(self.display_widget, title="LXMF Propagation Peers"), widget_style))
+        if hasattr(self, "peer_list") and self.peer_list:
+            pl = len(self.peer_list)
+        else:
+            pl = 0
+        urwid.WidgetWrap.__init__(self, urwid.AttrMap(urwid.LineBox(self.display_widget, title=f"LXMF Propagation Peers ({pl})"), widget_style))
 
     def keypress(self, size, key):
         if key == "up" and (self.no_content or self.ilb.first_item_is_selected()):
@@ -1576,13 +1615,13 @@ class LXMFPeers(urwid.WidgetWrap):
 
     def make_peer_widgets(self):
         widget_list = []
-        for peer_id in self.peer_list:
+        sorted_peers = sorted(self.peer_list, key=lambda pid: self.peer_list[pid].link_establishment_rate, reverse=True)
+        for peer_id in sorted_peers:
             peer = self.peer_list[peer_id]
             pe = LXMFPeerEntry(self.app, peer, self)
             pe.destination_hash = peer.destination_hash
             widget_list.append(pe)
 
-        # TODO: Sort list
         return widget_list
 
 class LXMFPeerEntry(urwid.WidgetWrap):
@@ -1598,7 +1637,7 @@ class LXMFPeerEntry(urwid.WidgetWrap):
             node_hash = RNS.Destination.hash_from_name_and_identity("nomadnetwork.node", node_identity)
             display_name = self.app.directory.alleged_display_str(node_hash)
             if display_name != None:
-                display_str += " "+str(display_name)
+                display_str = str(display_name)+"\n  "+display_str
 
         sym = g["sent"]
         style         = "list_unknown"
@@ -1608,10 +1647,14 @@ class LXMFPeerEntry(urwid.WidgetWrap):
         if hasattr(peer, "alive"):
             if peer.alive:
                 alive_string = "Available"
+                style = "list_normal"
+                focus_style = "list_focus"
             else:
                 alive_string = "Unresponsive"
+                style = "list_unresponsive"
+                focus_style = "list_focus_unresponsive"
 
-        widget = ListEntry(sym+" "+display_str+"\n  "+alive_string+", last heard "+pretty_date(int(peer.last_heard))+"\n  "+str(len(peer.unhandled_messages))+" unhandled LXMs")
+        widget = ListEntry(sym+" "+display_str+"\n  "+alive_string+", last heard "+pretty_date(int(peer.last_heard))+"\n  "+str(len(peer.unhandled_messages))+" unhandled LXMs, "+RNS.prettysize(peer.link_establishment_rate/8, "b")+"/s LER")
         # urwid.connect_signal(widget, "click", delegate.connect_node, node)
 
         self.display_widget = urwid.AttrMap(widget, style, focus_style)

nomadnet/vendor/Scrollable.py

@@ -107,6 +107,51 @@ class Scrollable(urwid.WidgetDecoration):
             if canv_full.cursor is not None:
                 # Full canvas contains the cursor, but scrolled out of view
                 self._forward_keypress = False
+                
+                # Reset cursor position on page/up down scrolling
+                try:
+                    if hasattr(ow, "automove_cursor_on_scroll") and ow.automove_cursor_on_scroll:
+                        pwi = 0
+                        ch = 0
+                        last_hidden = False
+                        first_visible = False
+                        for w,o in ow.contents:
+                            wcanv = w.render((maxcol,))
+                            wh = wcanv.rows()
+                            if wh:
+                                ch += wh
+
+                            if not last_hidden and ch >= self._trim_top:
+                                last_hidden = True
+
+                            elif last_hidden:
+                                if not first_visible:
+                                    first_visible = True
+
+                                if w.selectable():
+                                    ow.focus_item = pwi
+
+                                    st = None
+                                    nf = ow.get_focus()
+                                    if hasattr(nf, "key_timeout"):
+                                        st = nf
+                                    elif hasattr(nf, "original_widget"):
+                                        no = nf.original_widget
+                                        if hasattr(no, "original_widget"):
+                                            st = no.original_widget
+                                        else:
+                                            if hasattr(no, "key_timeout"):
+                                                st = no
+                                    
+                                    if st and hasattr(st, "key_timeout") and hasattr(st, "keypress") and callable(st.keypress):
+                                        st.keypress(None, None)
+                                    
+                                    break
+
+                            pwi += 1
+                except Exception as e:
+                    pass
+
             else:
                 # Original widget does not have a cursor, but may be selectable
 

nomadnet/vendor/quotes.py

@@ -3,4 +3,5 @@ quotes = [
     ("That's enough entropy for you my friend", "Unknown"),
     ("Any time two people connect online, it's financed by a third person who believes they can manipulate the first two", "Jaron Lanier"),
     ("The landscape of the future is set, but how one will march across it is not determined", "Terence McKenna")
+    ("Freedom originates in the division of power, despotism in its concentration.", "John Acton")
 ]

setup.py

@@ -5,6 +5,12 @@ exec(open("nomadnet/_version.py", "r").read())
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
+package_data = {
+"": [
+    "examples/messageboard/*",
+    ]
+}
+
 setuptools.setup(
     name="nomadnet",
     version=__version__,
@@ -15,6 +21,7 @@ setuptools.setup(
     long_description_content_type="text/markdown",
     url="https://github.com/markqvist/nomadnet",
     packages=setuptools.find_packages(),
+    package_data=package_data,
     classifiers=[
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: MIT License",
@@ -23,6 +30,6 @@ setuptools.setup(
     entry_points= {
         'console_scripts': ['nomadnet=nomadnet.nomadnet:main']
     },
-    install_requires=["rns>=0.4.3", "lxmf>=0.2.7", "urwid>=2.1.2", "qrcode"],
+    install_requires=["rns>=0.5.9", "lxmf>=0.3.3", "urwid>=2.1.2", "qrcode"],
     python_requires=">=3.6",
 )