Updated documentation and manual

RNS/Destination.py

@@ -23,6 +23,7 @@
 import os
 import math
 import time
+import threading
 import RNS
 
 from RNS.Cryptography import Fernet
@@ -152,8 +153,10 @@ class Destination:
         self.ratchets = None
         self.ratchets_path = None
         self.ratchet_interval = Destination.RATCHET_INTERVAL
+        self.ratchet_file_lock = threading.Lock()
         self.retained_ratchets = Destination.RATCHET_COUNT
         self.latest_ratchet_time = None
+        self.latest_ratchet_id = None
         self.__enforce_ratchets = False
         self.mtu = 0
 
@@ -195,11 +198,12 @@ class Destination:
 
     def _persist_ratchets(self):
         try:
-            packed_ratchets = umsgpack.packb(self.ratchets)
-            persisted_data = {"signature": self.sign(packed_ratchets), "ratchets": packed_ratchets}
-            ratchets_file = open(self.ratchets_path, "wb")
-            ratchets_file.write(umsgpack.packb(persisted_data))
-            ratchets_file.close()
+            with self.ratchet_file_lock:
+                packed_ratchets = umsgpack.packb(self.ratchets)
+                persisted_data = {"signature": self.sign(packed_ratchets), "ratchets": packed_ratchets}
+                ratchets_file = open(self.ratchets_path, "wb")
+                ratchets_file.write(umsgpack.packb(persisted_data))
+                ratchets_file.close()
         except Exception as e:
             self.ratchets = None
             self.ratchets_path = None
@@ -266,9 +270,6 @@ class Destination:
                 self.rotate_ratchets()
                 ratchet = RNS.Identity._ratchet_public_bytes(self.ratchets[0])
 
-                # TODO: Remove at some point
-                RNS.log(f"Including ratchet {RNS.prettyhexrep(RNS.Identity.truncated_hash(ratchet))} in announce", RNS.LOG_EXTREME)
-
             if app_data == None and self.default_app_data != None:
                 if isinstance(self.default_app_data, bytes):
                     app_data = self.default_app_data
@@ -401,6 +402,7 @@ class Destination:
             self.incoming_link_request(plaintext, packet)
         else:
             plaintext = self.decrypt(packet.data)
+            packet.ratchet_id = self.latest_ratchet_id
             if plaintext != None:
                 if packet.packet_type == RNS.Packet.DATA:
                     if self.callbacks.packet != None:
@@ -415,6 +417,29 @@ class Destination:
             if link != None:
                 self.links.append(link)
 
+    def _reload_ratchets(self, ratchets_path):
+        if os.path.isfile(ratchets_path):
+            with self.ratchet_file_lock:
+                try:
+                    ratchets_file = open(ratchets_path, "rb")
+                    persisted_data = umsgpack.unpackb(ratchets_file.read())
+                    if "signature" in persisted_data and "ratchets" in persisted_data:
+                        if self.identity.validate(persisted_data["signature"], persisted_data["ratchets"]):
+                            self.ratchets = umsgpack.unpackb(persisted_data["ratchets"])
+                            self.ratchets_path = ratchets_path
+                        else:
+                            raise KeyError("Invalid ratchet file signature")
+
+                except Exception as e:
+                    self.ratchets = None
+                    self.ratchets_path = None
+                    raise OSError("Could not read ratchet file contents for "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
+        else:
+            RNS.log("No existing ratchet data found, initialising new ratchet file for "+str(self), RNS.LOG_DEBUG)
+            self.ratchets = []
+            self.ratchets_path = ratchets_path
+            self._persist_ratchets()
+
     def enable_ratchets(self, ratchets_path):
         """
         Enables ratchets on the destination. When ratchets are enabled, Reticulum will automatically rotate
@@ -432,26 +457,7 @@ class Destination:
         """
         if ratchets_path != None:
             self.latest_ratchet_time = 0
-            if os.path.isfile(ratchets_path):
-                try:
-                    ratchets_file = open(ratchets_path, "rb")
-                    persisted_data = umsgpack.unpackb(ratchets_file.read())
-                    if "signature" in persisted_data and "ratchets" in persisted_data:
-                        if self.identity.validate(persisted_data["signature"], persisted_data["ratchets"]):
-                            self.ratchets = umsgpack.unpackb(persisted_data["ratchets"])
-                            self.ratchets_path = ratchets_path
-                        else:
-                            raise KeyError("Invalid ratchet file signature")
-
-                except Exception as e:
-                    self.ratchets = None
-                    self.ratchets_path = None
-                    raise OSError("Could not read ratchet file contents for "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
-            else:
-                RNS.log("No existing ratchet data found, initialising new ratchet file for "+str(self), RNS.LOG_DEBUG)
-                self.ratchets = []
-                self.ratchets_path = ratchets_path
-                self._persist_ratchets()
+            self._reload_ratchets(ratchets_path)
 
             # TODO: Remove at some point
             RNS.log("Ratchets enabled on "+str(self), RNS.LOG_DEBUG)
@@ -565,7 +571,10 @@ class Destination:
             return plaintext
 
         if self.type == Destination.SINGLE and self.identity != None:
-            return self.identity.encrypt(plaintext, ratchet=RNS.Identity.get_ratchet(self.hash))
+            selected_ratchet = RNS.Identity.get_ratchet(self.hash)
+            if selected_ratchet:
+                self.latest_ratchet_id = RNS.Identity.truncated_hash(selected_ratchet)
+            return self.identity.encrypt(plaintext, ratchet=selected_ratchet)
 
         if self.type == Destination.GROUP:
             if hasattr(self, "prv") and self.prv != None:
@@ -588,7 +597,28 @@ class Destination:
             return ciphertext
 
         if self.type == Destination.SINGLE and self.identity != None:
-            return self.identity.decrypt(ciphertext, ratchets=self.ratchets, enforce_ratchets=self.__enforce_ratchets)
+            if self.ratchets:
+                decrypted = None
+                try:
+                    decrypted = self.identity.decrypt(ciphertext, ratchets=self.ratchets, enforce_ratchets=self.__enforce_ratchets, ratchet_id_receiver=self)
+                except:
+                    decrypted = None
+
+                if not decrypted:
+                    try:
+                        RNS.log(f"Decryption with ratchets failed on {self}, reloading ratchets from storage and retrying", RNS.LOG_ERROR)
+                        self._reload_ratchets(self.ratchets_path)
+                        decrypted = self.identity.decrypt(ciphertext, ratchets=self.ratchets, enforce_ratchets=self.__enforce_ratchets, ratchet_id_receiver=self)
+                    except Exception as e:
+                        RNS.log(f"Decryption still failing after ratchet reload. The contained exception was: {e}", RNS.LOG_ERROR)
+                        raise e
+
+                    RNS.log("Decryption succeeded after ratchet reload", RNS.LOG_NOTICE)
+
+                return decrypted
+
+            else:
+                return self.identity.decrypt(ciphertext, ratchets=None, enforce_ratchets=self.__enforce_ratchets, ratchet_id_receiver=self)
 
         if self.type == Destination.GROUP:
             if hasattr(self, "prv") and self.prv != None:

RNS/Identity.py

@@ -324,11 +324,11 @@ class Identity:
         if not destination_hash in Identity.known_ratchets:
             ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
             hexhash = RNS.hexrep(destination_hash, delimit=False)
-            ratchet_path = f"{ratchetdir}/hexhash"
+            ratchet_path = f"{ratchetdir}/{hexhash}"
             if os.path.isfile(ratchet_path):
                 try:
                     ratchet_file = open(ratchet_path, "rb")
-                    ratchet_data = umsgpack.unpackb(ratchets_file.read())
+                    ratchet_data = umsgpack.unpackb(ratchet_file.read())
                     if time.time() < ratchet_data["received"]+Identity.RATCHET_EXPIRY and len(ratchet_data["ratchet"]) == Identity.RATCHETSIZE//8:
                         Identity.known_ratchets[destination_hash] = ratchet_data["ratchet"]
                     else:
@@ -631,8 +631,6 @@ class Identity:
             ephemeral_pub_bytes = ephemeral_key.public_key().public_bytes()
 
             if ratchet != None:
-                # TODO: Remove at some point
-                RNS.log(f"Encrypting with ratchet {RNS.prettyhexrep(RNS.Identity.truncated_hash(ratchet))}", RNS.LOG_EXTREME)
                 target_public_key = X25519PublicKey.from_public_bytes(ratchet)
             else:
                 target_public_key = self.pub
@@ -655,7 +653,7 @@ class Identity:
             raise KeyError("Encryption failed because identity does not hold a public key")
 
 
-    def decrypt(self, ciphertext_token, ratchets=None, enforce_ratchets=False):
+    def decrypt(self, ciphertext_token, ratchets=None, enforce_ratchets=False, ratchet_id_receiver=None):
         """
         Decrypts information for the identity.
 
@@ -675,6 +673,7 @@ class Identity:
                         for ratchet in ratchets:
                             try:
                                 ratchet_prv = X25519PrivateKey.from_private_bytes(ratchet)
+                                ratchet_id = Identity.truncated_hash(ratchet_prv.public_key().public_bytes())
                                 shared_key = ratchet_prv.exchange(peer_pub)
                                 derived_key = RNS.Cryptography.hkdf(
                                     length=32,
@@ -685,9 +684,8 @@ class Identity:
 
                                 fernet = Fernet(derived_key)
                                 plaintext = fernet.decrypt(ciphertext)
-                                
-                                # TODO: Remove at some point
-                                RNS.log(f"Decrypted with ratchet {RNS.prettyhexrep(RNS.Identity.truncated_hash(ratchet_prv.public_key().public_bytes()))}", RNS.LOG_EXTREME)
+                                if ratchet_id_receiver:
+                                    ratchet_id_receiver.latest_ratchet_id = ratchet_id
                                 
                                 break
                             
@@ -696,6 +694,8 @@ class Identity:
 
                     if enforce_ratchets and plaintext == None:
                         RNS.log("Decryption with ratchet enforcement by "+RNS.prettyhexrep(self.hash)+" failed. Dropping packet.", RNS.LOG_DEBUG)
+                        if ratchet_id_receiver:
+                            ratchet_id_receiver.latest_ratchet_id = None
                         return None
 
                     if plaintext == None:
@@ -709,9 +709,13 @@ class Identity:
 
                         fernet = Fernet(derived_key)
                         plaintext = fernet.decrypt(ciphertext)
+                        if ratchet_id_receiver:
+                            ratchet_id_receiver.latest_ratchet_id = None
 
                 except Exception as e:
                     RNS.log("Decryption by "+RNS.prettyhexrep(self.hash)+" failed: "+str(e), RNS.LOG_DEBUG)
+                    if ratchet_id_receiver:
+                        ratchet_id_receiver.latest_ratchet_id = None
                     
                 return plaintext;
             else:

RNS/Link.py

@@ -775,6 +775,7 @@ class Link:
                     should_query = False
                     if packet.context == RNS.Packet.NONE:
                         plaintext = self.decrypt(packet.data)
+                        packet.ratchet_id = self.link_id
                         if plaintext != None:
                             if self.callbacks.packet != None:
                                 thread = threading.Thread(target=self.callbacks.packet, args=(plaintext, packet))

RNS/Packet.py

@@ -140,6 +140,7 @@ class Packet:
         self.MTU         = RNS.Reticulum.MTU
         self.sent_at     = None
         self.packet_hash = None
+        self.ratchet_id  = None
 
         self.attached_interface = attached_interface
         self.receiving_interface = None
@@ -195,6 +196,8 @@ class Packet:
                     # In all other cases, we encrypt the packet
                     # with the destination's encryption method
                     self.ciphertext = self.destination.encrypt(self.data)
+                    if hasattr(self.destination, "latest_ratchet_id"):
+                        self.ratchet_id = self.destination.latest_ratchet_id
 
             if self.header_type == Packet.HEADER_2:
                 if self.transport_id != None:
@@ -418,6 +421,7 @@ class PacketReceipt:
                         except Exception as e:
                             RNS.log("An error occurred while evaluating external delivery callback for "+str(link), RNS.LOG_ERROR)
                             RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+                            RNS.trace_exception(e)
                             
                     return True
                 else:

RNS/Transport.py

@@ -52,16 +52,16 @@ class Transport:
     Maximum amount of hops that Reticulum will transport a packet.
     """
     
-    PATHFINDER_R                = 1          # Retransmit retries
-    PATHFINDER_G                = 5          # Retry grace period
-    PATHFINDER_RW               = 0.5        # Random window for announce rebroadcast
-    PATHFINDER_E                = 60*60*24*7 # Path expiration of one week
-    AP_PATH_TIME                = 60*60*24   # Path expiration of one day for Access Point paths
-    ROAMING_PATH_TIME           = 60*60*6    # Path expiration of 6 hours for Roaming paths
+    PATHFINDER_R                = 1            # Retransmit retries
+    PATHFINDER_G                = 5            # Retry grace period
+    PATHFINDER_RW               = 0.5          # Random window for announce rebroadcast
+    PATHFINDER_E                = 60*60*24*7   # Path expiration of one week
+    AP_PATH_TIME                = 60*60*24     # Path expiration of one day for Access Point paths
+    ROAMING_PATH_TIME           = 60*60*6      # Path expiration of 6 hours for Roaming paths
 
     # TODO: Calculate an optimal number for this in
     # various situations
-    LOCAL_REBROADCASTS_MAX      = 2          # How many local rebroadcasts of an announce is allowed
+    LOCAL_REBROADCASTS_MAX      = 2            # How many local rebroadcasts of an announce is allowed
 
     PATH_REQUEST_TIMEOUT        = 15           # Default timuout for client path requests in seconds
     PATH_REQUEST_GRACE          = 0.4          # Grace time before a path announcement is made, allows directly reachable peers to respond first

docs/manual/genindex.html

@@ -293,6 +293,8 @@
         <li><a href="reference.html#RNS.Buffer.create_reader">create_reader() (RNS.Buffer static method)</a>
 </li>
         <li><a href="reference.html#RNS.Buffer.create_writer">create_writer() (RNS.Buffer static method)</a>
+</li>
+        <li><a href="reference.html#RNS.Identity.current_ratchet_id">current_ratchet_id() (RNS.Identity static method)</a>
 </li>
         <li><a href="reference.html#RNS.Identity.CURVE">CURVE (RNS.Identity attribute)</a>
 

docs/manual/reference.html

@@ -417,7 +417,7 @@ for addressable hashes and other purposes. Non-configurable.</p>
 <dd class="field-odd"><p><strong>data</strong> – Data to be hashed as <em>bytes</em>.</p>
 </dd>
 <dt class="field-even">Returns<span class="colon">:</span></dt>
-<dd class="field-even"><p>SHA-256 hash as <em>bytes</em></p>
+<dd class="field-even"><p>SHA-256 hash as <em>bytes</em>.</p>
 </dd>
 </dl>
 </dd></dl>
@@ -431,7 +431,7 @@ for addressable hashes and other purposes. Non-configurable.</p>
 <dd class="field-odd"><p><strong>data</strong> – Data to be hashed as <em>bytes</em>.</p>
 </dd>
 <dt class="field-even">Returns<span class="colon">:</span></dt>
-<dd class="field-even"><p>Truncated SHA-256 hash as <em>bytes</em></p>
+<dd class="field-even"><p>Truncated SHA-256 hash as <em>bytes</em>.</p>
 </dd>
 </dl>
 </dd></dl>
@@ -445,7 +445,21 @@ for addressable hashes and other purposes. Non-configurable.</p>
 <dd class="field-odd"><p><strong>data</strong> – Data to be hashed as <em>bytes</em>.</p>
 </dd>
 <dt class="field-even">Returns<span class="colon">:</span></dt>
-<dd class="field-even"><p>Truncated SHA-256 hash of random data as <em>bytes</em></p>
+<dd class="field-even"><p>Truncated SHA-256 hash of random data as <em>bytes</em>.</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="RNS.Identity.current_ratchet_id">
+<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">current_ratchet_id</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">destination_hash</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#RNS.Identity.current_ratchet_id" title="Permalink to this definition">#</a></dt>
+<dd><p>Get the ID of the currently used ratchet key for a given destination hash</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>destination_hash</strong> – A destination hash as <em>bytes</em>.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>A ratchet ID as <em>bytes</em> or <em>None</em>.</p>
 </dd>
 </dl>
 </dd></dl>
@@ -563,7 +577,7 @@ communication for the identity. Be very careful with this method.</p>
 
 <dl class="py method">
 <dt class="sig sig-object py" id="RNS.Identity.decrypt">
-<span class="sig-name descname"><span class="pre">decrypt</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">ciphertext_token</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">ratchets</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">enforce_ratchets</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#RNS.Identity.decrypt" title="Permalink to this definition">#</a></dt>
+<span class="sig-name descname"><span class="pre">decrypt</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">ciphertext_token</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">ratchets</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">enforce_ratchets</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">ratchet_id_receiver</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#RNS.Identity.decrypt" title="Permalink to this definition">#</a></dt>
 <dd><p>Decrypts information for the identity.</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>
@@ -2058,6 +2072,7 @@ will announce it.</p>
 <li><a class="reference internal" href="#RNS.Identity.full_hash"><code class="docutils literal notranslate"><span class="pre">full_hash()</span></code></a></li>
 <li><a class="reference internal" href="#RNS.Identity.truncated_hash"><code class="docutils literal notranslate"><span class="pre">truncated_hash()</span></code></a></li>
 <li><a class="reference internal" href="#RNS.Identity.get_random_hash"><code class="docutils literal notranslate"><span class="pre">get_random_hash()</span></code></a></li>
+<li><a class="reference internal" href="#RNS.Identity.current_ratchet_id"><code class="docutils literal notranslate"><span class="pre">current_ratchet_id()</span></code></a></li>
 <li><a class="reference internal" href="#RNS.Identity.from_bytes"><code class="docutils literal notranslate"><span class="pre">from_bytes()</span></code></a></li>
 <li><a class="reference internal" href="#RNS.Identity.from_file"><code class="docutils literal notranslate"><span class="pre">from_file()</span></code></a></li>
 <li><a class="reference internal" href="#RNS.Identity.to_file"><code class="docutils literal notranslate"><span class="pre">to_file()</span></code></a></li>