From 9ef10a7b3e2e861a40f92bdc0e75fb4965d9a103 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 5 Sep 2024 15:02:22 +0200 Subject: [PATCH] Expanded and documented ratchet API --- RNS/Destination.py | 170 ++++++++++++++++++++++++++++++++------------- RNS/Identity.py | 88 +++++++++++++++++------ RNS/Reticulum.py | 1 + 3 files changed, 189 insertions(+), 70 deletions(-) diff --git a/RNS/Destination.py b/RNS/Destination.py index b7d510a..ec2ebb7 100755 --- a/RNS/Destination.py +++ b/RNS/Destination.py @@ -72,7 +72,16 @@ class Destination: directions = [IN, OUT] PR_TAG_WINDOW = 30 - RATCHET_COUNT = 512 + + RATCHET_COUNT = 512 + """ + The default number of generated ratchet keys a destination will retain, if it has ratchets enabled. + """ + + RATCHET_INTERVAL = 30*60 + """ + The minimum interval between rotating ratchet keys, in seconds. + """ @staticmethod def expand_name(identity, app_name, *aspects): @@ -142,6 +151,10 @@ class Destination: self.proof_strategy = Destination.PROVE_NONE self.ratchets = None self.ratchets_path = None + self.ratchet_interval = Destination.RATCHET_INTERVAL + self.retained_ratchets = Destination.RATCHET_COUNT + self.latest_ratchet_time = None + self.__enforce_ratchets = False self.mtu = 0 self.path_responses = {} @@ -175,36 +188,12 @@ class Destination: """ return "<"+self.name+"/"+self.hexhash+">" - def enable_ratchets(self, ratchets_path): - if ratchets_path != None: - if os.path.isfile(ratchets_path): - try: - ratchets_file = open(ratchets_path, "rb") - persisted_data = umsgpack.unpackb(ratchets_file.read()) - if "signature" in persisted_data and "ratchets" in persisted_data: - if self.identity.validate(persisted_data["signature"], persisted_data["ratchets"]): - self.ratchets = umsgpack.unpackb(persisted_data["ratchets"]) - self.ratchets_path = ratchets_path - else: - raise KeyError("Invalid ratchet file signature") + def _clean_ratchets(self): + if self.ratchets != None: + if len (self.ratchets) > self.retained_ratchets: + self.ratchets = self.ratchets[:Destination.RATCHET_COUNT] - except Exception as e: - self.ratchets = None - self.ratchets_path = None - raise OSError("Could not read ratchet file contents for "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR) - else: - RNS.log("No existing ratchet data found, initialising new ratchet file for "+str(self), RNS.LOG_DEBUG) - self.ratchets = [] - self.ratchets_path = ratchets_path - self.persist_ratchets() - - RNS.log("Ratchets enabled on "+str(self), RNS.LOG_DEBUG) # TODO: Remove - return True - - else: - raise ValueError("No ratchet file path specified for "+str(self)) - - def persist_ratchets(self): + def _persist_ratchets(self): try: packed_ratchets = umsgpack.packb(self.ratchets) persisted_data = {"signature": self.sign(packed_ratchets), "ratchets": packed_ratchets} @@ -218,15 +207,20 @@ class Destination: def rotate_ratchets(self): if self.ratchets != None: - RNS.log("Rotating ratchets for "+str(self), RNS.LOG_DEBUG) # TODO: Remove - new_ratchet = RNS.Identity._generate_ratchet() - self.ratchets.insert(0, new_ratchet) - if len (self.ratchets) > Destination.RATCHET_COUNT: - self.ratchets = self.ratchets[:Destination.RATCHET_COUNT] - self.persist_ratchets() + now = time.time() + if now > self.latest_ratchet_time+self.ratchet_interval: + RNS.log("Rotating ratchets for "+str(self), RNS.LOG_DEBUG) + new_ratchet = RNS.Identity._generate_ratchet() + self.ratchets.insert(0, new_ratchet) + self.latest_ratchet_time = now + self._clean_ratchets() + self._persist_ratchets() + return True else: raise SystemError("Cannot rotate ratchet on "+str(self)+", ratchets are not enabled") + return False + def announce(self, app_data=None, path_response=False, attached_interface=None, tag=None, send=True): """ Creates an announce packet for this destination and broadcasts it on all @@ -272,8 +266,8 @@ class Destination: self.rotate_ratchets() ratchet = RNS.Identity._ratchet_public_bytes(self.ratchets[0]) - # TODO: Remove debug output - RNS.log(f"Including ratchet {RNS.prettyhexrep(RNS.Identity.truncated_hash(ratchet))} in announce", RNS.LOG_DEBUG) + # TODO: Remove at some point + RNS.log(f"Including ratchet {RNS.prettyhexrep(RNS.Identity.truncated_hash(ratchet))} in announce", RNS.LOG_EXTREME) if app_data == None and self.default_app_data != None: if isinstance(self.default_app_data, bytes): @@ -366,7 +360,6 @@ class Destination: else: self.proof_strategy = proof_strategy - def register_request_handler(self, path, response_generator = None, allow = ALLOW_NONE, allowed_list = None): """ Registers a request handler. @@ -388,7 +381,6 @@ class Destination: request_handler = [path, response_generator, allow, allowed_list] self.request_handlers[path_hash] = request_handler - def deregister_request_handler(self, path): """ Deregisters a request handler. @@ -403,8 +395,6 @@ class Destination: else: return False - - def receive(self, packet): if packet.packet_type == RNS.Packet.LINKREQUEST: plaintext = packet.data @@ -419,13 +409,99 @@ class Destination: except Exception as e: RNS.log("Error while executing receive callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR) - def incoming_link_request(self, data, packet): if self.accept_link_requests: link = RNS.Link.validate_request(self, data, packet) if link != None: self.links.append(link) + def enable_ratchets(self, ratchets_path): + """ + Enables ratchets on the destination. When ratchets are enabled, Reticulum will automatically rotate + the keys used to encrypt packets to this destination, and include the latest ratchet key in announces. + + Enabling ratchets on a destination will provide forward secrecy for packets sent to that destination, + even when sent outside a ``Link``. The normal Reticulum ``Link`` establishment procedure already performs + its own ephemeral key exchange for each link establishment, which means that ratchets are not necessary + to provide forward secrecy for links. + + Enabling ratchets will have a small impact on announce size, adding 32 bytes to every sent announce. + + :param ratchets_path: The path to a file to store ratchet data in. + :returns: True if the operation succeeded, otherwise False. + """ + if ratchets_path != None: + self.latest_ratchet_time = 0 + if os.path.isfile(ratchets_path): + try: + ratchets_file = open(ratchets_path, "rb") + persisted_data = umsgpack.unpackb(ratchets_file.read()) + if "signature" in persisted_data and "ratchets" in persisted_data: + if self.identity.validate(persisted_data["signature"], persisted_data["ratchets"]): + self.ratchets = umsgpack.unpackb(persisted_data["ratchets"]) + self.ratchets_path = ratchets_path + else: + raise KeyError("Invalid ratchet file signature") + + except Exception as e: + self.ratchets = None + self.ratchets_path = None + raise OSError("Could not read ratchet file contents for "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + else: + RNS.log("No existing ratchet data found, initialising new ratchet file for "+str(self), RNS.LOG_DEBUG) + self.ratchets = [] + self.ratchets_path = ratchets_path + self._persist_ratchets() + + # TODO: Remove at some point + RNS.log("Ratchets enabled on "+str(self), RNS.LOG_DEBUG) + return True + + else: + raise ValueError("No ratchet file path specified for "+str(self)) + + def enforce_ratchets(self): + """ + When ratchet enforcement is enabled, this destination will never accept packets that use its + base Identity key for encryption, but only accept packets encrypted with one of the retained + ratchet keys. + """ + if self.ratchets != None: + self.__enforce_ratchets = True + RNS.log("Ratchets enforced on "+str(self), RNS.LOG_DEBUG) + return True + else: + return False + + def set_retained_ratchets(self, retained_ratchets): + """ + Sets the number of previously generated ratchet keys this destination will retain, + and try to use when decrypting incoming packets. Defaults to ``Destination.RATCHET_COUNT``. + + :param retained_ratchets: The number of generated ratchets to retain. + :returns: True if the operation succeeded, False if not. + """ + if isinstance(retained_ratchets, int) and retained_ratchets > 0: + self.retained_ratchets = retained_ratchets + self._clean_ratchets() + return True + else: + return False + + def set_ratchet_interval(self, interval): + """ + Sets the minimum interval in seconds between ratchet key rotation. + Defaults to ``Destination.RATCHET_INTERVAL``. + + :param interval: The minimum interval in seconds. + :returns: True if the operation succeeded, False if not. + """ + if isinstance(interval, int) and interval > 0: + self.ratchet_interval = interval + return True + else: + return False + def create_keys(self): """ For a ``RNS.Destination.GROUP`` type destination, creates a new symmetric key. @@ -442,7 +518,6 @@ class Destination: self.prv_bytes = Fernet.generate_key() self.prv = Fernet(self.prv_bytes) - def get_private_key(self): """ For a ``RNS.Destination.GROUP`` type destination, returns the symmetric private key. @@ -456,7 +531,6 @@ class Destination: else: return self.prv_bytes - def load_private_key(self, key): """ For a ``RNS.Destination.GROUP`` type destination, loads a symmetric private key. @@ -480,7 +554,6 @@ class Destination: else: raise TypeError("A single destination holds keys through an Identity instance") - def encrypt(self, plaintext): """ Encrypts information for ``RNS.Destination.SINGLE`` or ``RNS.Destination.GROUP`` type destination. @@ -504,8 +577,6 @@ class Destination: else: raise ValueError("No private key held by GROUP destination. Did you create or load one?") - - def decrypt(self, ciphertext): """ Decrypts information for ``RNS.Destination.SINGLE`` or ``RNS.Destination.GROUP`` type destination. @@ -517,7 +588,7 @@ class Destination: return ciphertext if self.type == Destination.SINGLE and self.identity != None: - return self.identity.decrypt(ciphertext, ratchets=self.ratchets) + return self.identity.decrypt(ciphertext, ratchets=self.ratchets, enforce_ratchets=self.__enforce_ratchets) if self.type == Destination.GROUP: if hasattr(self, "prv") and self.prv != None: @@ -529,7 +600,6 @@ class Destination: else: raise ValueError("No private key held by GROUP destination. Did you create or load one?") - def sign(self, message): """ Signs information for ``RNS.Destination.SINGLE`` type destination. diff --git a/RNS/Identity.py b/RNS/Identity.py index 8ce5633..4fb3d9a 100644 --- a/RNS/Identity.py +++ b/RNS/Identity.py @@ -26,6 +26,7 @@ import RNS import time import atexit import hashlib +import threading from .vendor import umsgpack as umsgpack @@ -49,11 +50,20 @@ class Identity: KEYSIZE = 256*2 """ - X25519 key size in bits. A complete key is the concatenation of a 256 bit encryption key, and a 256 bit signing key. + X.25519 key size in bits. A complete key is the concatenation of a 256 bit encryption key, and a 256 bit signing key. """ RATCHETSIZE = 256 + """ + X.25519 ratchet key size in bits. + """ + RATCHET_EXPIRY = 60*60*24*30 + """ + The expiry time for received ratchets in seconds, defaults to 30 days. Reticulum will always use the most recently + announced ratchet, and remember it for up to ``RATCHET_EXPIRY`` since receiving it, after which it will be discarded. + If a newer ratchet is announced in the meantime, it will be replace the already known ratchet. + """ # Non-configurable constants FERNET_OVERHEAD = RNS.Cryptography.Fernet.FERNET_OVERHEAD @@ -72,6 +82,8 @@ class Identity: known_destinations = {} known_ratchets = {} + ratchet_persist_lock = threading.Lock() + @staticmethod def remember(packet_hash, destination_hash, public_key, app_data = None): if len(public_key) != Identity.KEYSIZE//8: @@ -237,33 +249,64 @@ class Identity: @staticmethod def _remember_ratchet(destination_hash, ratchet): - RNS.log(f"Remembering ratchet {RNS.prettyhexrep(Identity.truncated_hash(ratchet))} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) # TODO: Remove + # TODO: Remove at some point + RNS.log(f"Remembering ratchet {RNS.prettyhexrep(Identity.truncated_hash(ratchet))} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_EXTREME) try: Identity.known_ratchets[destination_hash] = ratchet - hexhash = RNS.hexrep(destination_hash, delimit=False) - ratchet_data = {"ratchet": ratchet, "received": time.time()} - ratchetdir = RNS.Reticulum.storagepath+"/ratchets" - - if not os.path.isdir(ratchetdir): - os.makedirs(ratchetdir) + if not RNS.Transport.owner.is_connected_to_shared_instance: + def persist_job(): + with Identity.ratchet_persist_lock: + hexhash = RNS.hexrep(destination_hash, delimit=False) + ratchet_data = {"ratchet": ratchet, "received": time.time()} - outpath = f"{ratchetdir}/{hexhash}.out" - finalpath = f"{ratchetdir}/{hexhash}" - ratchet_file = open(outpath, "wb") - ratchet_file.write(umsgpack.packb(ratchet_data)) - ratchet_file.close() - os.rename(outpath, finalpath) + ratchetdir = RNS.Reticulum.storagepath+"/ratchets" + + if not os.path.isdir(ratchetdir): + os.makedirs(ratchetdir) + + outpath = f"{ratchetdir}/{hexhash}.out" + finalpath = f"{ratchetdir}/{hexhash}" + ratchet_file = open(outpath, "wb") + ratchet_file.write(umsgpack.packb(ratchet_data)) + ratchet_file.close() + os.rename(outpath, finalpath) + + + threading.Thread(target=persist_job, daemon=True).start() except Exception as e: RNS.log(f"Could not persist ratchet for {RNS.prettyhexrep(destination_hash)} to storage.", RNS.LOG_ERROR) RNS.log(f"The contained exception was: {e}") RNS.trace_exception(e) + @staticmethod + def _clean_ratchets(): + RNS.log("Cleaning ratchets...", RNS.LOG_DEBUG) + try: + now = time.time() + ratchetdir = RNS.Reticulum.storagepath+"/ratchets" + for filename in os.listdir(ratchetdir): + try: + expired = False + with open(f"{ratchetdir}/{filename}", "rb") as rf: + ratchet_data = umsgpack.unpackb(rf.read()) + if now > ratchet_data["received"]+Identity.RATCHET_EXPIRY: + expired = True + + if expired: + os.unlink(f"{ratchetdir}/{filename}") + + except Exception as e: + RNS.log(f"An error occurred while cleaning ratchets, in the processing of {ratchetdir}/{filename}.", RNS.LOG_ERROR) + RNS.log(f"The contained exception was: {e}", RNS.LOG_ERROR) + + except Exception as e: + RNS.log(f"An error occurred while cleaning ratchets. The contained exception was: {e}", RNS.LOG_ERROR) + @staticmethod def get_ratchet(destination_hash): if not destination_hash in Identity.known_ratchets: - RNS.log(f"Trying to load ratchet for {RNS.prettyhexrep(destination_hash)} from storage") # TODO: Remove ratchetdir = RNS.Reticulum.storagepath+"/ratchets" hexhash = RNS.hexrep(destination_hash, delimit=False) ratchet_path = f"{ratchetdir}/hexhash" @@ -284,6 +327,7 @@ class Identity: if destination_hash in Identity.known_ratchets: return Identity.known_ratchets[destination_hash] else: + RNS.log(f"Could not load ratchet for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) return None @staticmethod @@ -572,7 +616,8 @@ class Identity: ephemeral_pub_bytes = ephemeral_key.public_key().public_bytes() if ratchet != None: - RNS.log(f"Encrypting with ratchet {RNS.prettyhexrep(RNS.Identity.truncated_hash(ratchet))}", RNS.LOG_DEBUG) # TODO: Remove + # TODO: Remove at some point + RNS.log(f"Encrypting with ratchet {RNS.prettyhexrep(RNS.Identity.truncated_hash(ratchet))}", RNS.LOG_EXTREME) target_public_key = X25519PublicKey.from_public_bytes(ratchet) else: target_public_key = self.pub @@ -595,7 +640,7 @@ class Identity: raise KeyError("Encryption failed because identity does not hold a public key") - def decrypt(self, ciphertext_token, ratchets=None): + def decrypt(self, ciphertext_token, ratchets=None, enforce_ratchets=False): """ Decrypts information for the identity. @@ -626,14 +671,17 @@ class Identity: fernet = Fernet(derived_key) plaintext = fernet.decrypt(ciphertext) - # TODO: Remove - RNS.log(f"Decrypted with ratchet {RNS.prettyhexrep(RNS.Identity.truncated_hash(ratchet_prv.public_key().public_bytes()))}", RNS.LOG_DEBUG) + # TODO: Remove at some point + RNS.log(f"Decrypted with ratchet {RNS.prettyhexrep(RNS.Identity.truncated_hash(ratchet_prv.public_key().public_bytes()))}", RNS.LOG_EXTREME) break except Exception as e: pass - # RNS.log("Decryption using this ratchet failed", RNS.LOG_DEBUG) # TODO: Remove + + if enforce_ratchets and plaintext == None: + RNS.log("Decryption with ratchet enforcement by "+RNS.prettyhexrep(self.hash)+" failed. Dropping packet.", RNS.LOG_DEBUG) + return None if plaintext == None: shared_key = self.prv.exchange(peer_pub) diff --git a/RNS/Reticulum.py b/RNS/Reticulum.py index d28ca08..6ebb770 100755 --- a/RNS/Reticulum.py +++ b/RNS/Reticulum.py @@ -295,6 +295,7 @@ class Reticulum: def __start_jobs(self): if self.jobs_thread == None: + RNS.Identity._clean_ratchets() self.jobs_thread = threading.Thread(target=self.__jobs) self.jobs_thread.daemon = True self.jobs_thread.start()