William/Reticulum
1
0
Fork 0
mirror of https://github.com/markqvist/Reticulum.git synced 2025-01-18 16:30:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve resource transfer throughput on high-MTU links

Browse Source
This commit is contained in:
Mark Qvist 2025-01-11 13:22:18 +01:00
parent bf6e73e163
commit f5cf438abd
6 changed files with 43 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
RNS/Interfaces/LocalInterface.py
View File

@ -57,6 +57,7 @@ class LocalClientInterface(Interface):
super().__init__() super().__init__()
self.HW_MTU = 32768 self.HW_MTU = 32768
# self.HW_MTU = 500 # TODO: Remove debug
self.online = False self.online = False

15
RNS/Link.py
View File

@ -139,13 +139,14 @@ class Link:
link.set_link_id(packet) link.set_link_id(packet)
if len(data) == Link.ECPUBSIZE+Link.LINK_MTU_SIZE: if len(data) == Link.ECPUBSIZE+Link.LINK_MTU_SIZE:
RNS.log("Link request includes MTU signalling") # TODO: Remove debug RNS.log("Link request includes MTU signalling", RNS.LOG_DEBUG) # TODO: Remove debug
try: try:
link.mtu = Link.mtu_from_lr_packet(packet) or Reticulum.MTU link.mtu = Link.mtu_from_lr_packet(packet) or Reticulum.MTU
except Exception as e: except Exception as e:
RNS.trace_exception(e) RNS.trace_exception(e)
link.mtu = RNS.Reticulum.MTU link.mtu = RNS.Reticulum.MTU
link.update_mdu()
link.destination = packet.destination link.destination = packet.destination
link.establishment_timeout = Link.ESTABLISHMENT_TIMEOUT_PER_HOP * max(1, packet.hops) + Link.KEEPALIVE link.establishment_timeout = Link.ESTABLISHMENT_TIMEOUT_PER_HOP * max(1, packet.hops) + Link.KEEPALIVE
link.establishment_cost += len(packet.raw) link.establishment_cost += len(packet.raw)
@ -247,9 +248,9 @@ class Link:
if self.initiator: if self.initiator:
link_mtu = b"" link_mtu = b""
nh_hw_mtu = RNS.Transport.next_hop_interface_hw_mtu(destination.hash) nh_hw_mtu = RNS.Transport.next_hop_interface_hw_mtu(destination.hash)
if nh_hw_mtu: if RNS.Reticulum.LINK_MTU_DISCOVERY and nh_hw_mtu:
link_mtu = Link.mtu_bytes(nh_hw_mtu) link_mtu = Link.mtu_bytes(nh_hw_mtu)
RNS.log(f"Signalling link MTU of {RNS.prettysize(nh_hw_mtu)} for link") # TODO: Remove debug RNS.log(f"Signalling link MTU of {RNS.prettysize(nh_hw_mtu)} for link", RNS.LOG_DEBUG) # TODO: Remove debug
self.request_data = self.pub_bytes+self.sig_pub_bytes+link_mtu self.request_data = self.pub_bytes+self.sig_pub_bytes+link_mtu
self.packet = RNS.Packet(destination, self.request_data, packet_type=RNS.Packet.LINKREQUEST) self.packet = RNS.Packet(destination, self.request_data, packet_type=RNS.Packet.LINKREQUEST)
self.packet.pack() self.packet.pack()
@ -330,7 +331,7 @@ class Link:
confirmed_mtu = Link.mtu_from_lp_packet(packet) confirmed_mtu = Link.mtu_from_lp_packet(packet)
mtu_bytes = Link.mtu_bytes(confirmed_mtu) mtu_bytes = Link.mtu_bytes(confirmed_mtu)
packet.data = packet.data[:RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2] packet.data = packet.data[:RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2]
RNS.log(f"Destination confirmed link MTU of {RNS.prettysize(confirmed_mtu)}") # TODO: Remove debug RNS.log(f"Destination confirmed link MTU of {RNS.prettysize(confirmed_mtu)}", RNS.LOG_DEBUG) # TODO: Remove debug
if self.initiator and len(packet.data) == RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2: if self.initiator and len(packet.data) == RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2:
peer_pub_bytes = packet.data[RNS.Identity.SIGLENGTH//8:RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2] peer_pub_bytes = packet.data[RNS.Identity.SIGLENGTH//8:RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2]
@ -349,6 +350,8 @@ class Link:
self.rtt = time.time() - self.request_time self.rtt = time.time() - self.request_time
self.attached_interface = packet.receiving_interface self.attached_interface = packet.receiving_interface
self.__remote_identity = self.destination.identity self.__remote_identity = self.destination.identity
self.mtu = confirmed_mtu or RNS.Reticulum.MTU
self.update_mdu()
self.status = Link.ACTIVE self.status = Link.ACTIVE
self.activated_at = time.time() self.activated_at = time.time()
self.last_proof = self.activated_at self.last_proof = self.activated_at
@ -448,6 +451,10 @@ class Link:
) )
def update_mdu(self):
self.mdu = self.mtu - RNS.Reticulum.HEADER_MAXSIZE - RNS.Reticulum.IFAC_MIN_SIZE
RNS.log(f"Link MDU updated to {self.mdu}", RNS.LOG_DEBUG) # TODO: Remove debug
def rtt_packet(self, packet): def rtt_packet(self, packet):
try: try:
measured_rtt = time.time() - self.request_time measured_rtt = time.time() - self.request_time

4
RNS/Packet.py
View File

@ -137,7 +137,11 @@ class Packet:
self.fromPacked = True self.fromPacked = True
self.create_receipt = False self.create_receipt = False
if destination and destination.type == RNS.Destination.LINK:
self.MTU = destination.mtu
else:
self.MTU = RNS.Reticulum.MTU self.MTU = RNS.Reticulum.MTU
self.sent_at = None self.sent_at = None
self.packet_hash = None self.packet_hash = None
self.ratchet_id = None self.ratchet_id = None

14
RNS/Resource.py
View File

@ -163,7 +163,7 @@ class Resource:
resource.initiator = False resource.initiator = False
resource.callback = callback resource.callback = callback
resource.__progress_callback = progress_callback resource.__progress_callback = progress_callback
resource.total_parts = int(math.ceil(resource.size/float(Resource.SDU))) resource.total_parts = int(math.ceil(resource.size/float(resource.sdu)))
resource.received_count = 0 resource.received_count = 0
resource.outstanding_parts = 0 resource.outstanding_parts = 0
resource.parts = [None] * resource.total_parts resource.parts = [None] * resource.total_parts
@ -209,6 +209,7 @@ class Resource:
except Exception as e: except Exception as e:
RNS.log("Could not decode resource advertisement, dropping resource", RNS.LOG_DEBUG) RNS.log("Could not decode resource advertisement, dropping resource", RNS.LOG_DEBUG)
RNS.trace_exception(e) # TODO: Remove debug
return None return None
# Create a resource for transmission to a remote destination # Create a resource for transmission to a remote destination
@ -271,6 +272,7 @@ class Resource:
self.status = Resource.NONE self.status = Resource.NONE
self.link = link self.link = link
self.sdu = link.mdu or Resource.SDU
self.max_retries = Resource.MAX_RETRIES self.max_retries = Resource.MAX_RETRIES
self.max_adv_retries = Resource.MAX_ADV_RETRIES self.max_adv_retries = Resource.MAX_ADV_RETRIES
self.retries_left = self.max_retries self.retries_left = self.max_retries
@ -290,6 +292,7 @@ class Resource:
self.very_slow_rate_rounds = 0 self.very_slow_rate_rounds = 0
self.request_id = request_id self.request_id = request_id
self.is_response = is_response self.is_response = is_response
self.auto_compress = auto_compress
self.req_hashlist = [] self.req_hashlist = []
self.receiver_min_consecutive_height = 0 self.receiver_min_consecutive_height = 0
@ -346,7 +349,7 @@ class Resource:
self.size = len(self.data) self.size = len(self.data)
self.sent_parts = 0 self.sent_parts = 0
hashmap_entries = int(math.ceil(self.size/float(Resource.SDU))) hashmap_entries = int(math.ceil(self.size/float(self.sdu)))
self.total_parts = hashmap_entries self.total_parts = hashmap_entries
hashmap_ok = False hashmap_ok = False
@ -368,7 +371,7 @@ class Resource:
self.hashmap = b"" self.hashmap = b""
collision_guard_list = [] collision_guard_list = []
for i in range(0,hashmap_entries): for i in range(0,hashmap_entries):
data = self.data[i*Resource.SDU:(i+1)*Resource.SDU] data = self.data[i*self.sdu:(i+1)*self.sdu]
map_hash = self.get_map_hash(data) map_hash = self.get_map_hash(data)
if map_hash in collision_guard_list: if map_hash in collision_guard_list:
@ -647,6 +650,7 @@ class Resource:
request_id = self.request_id, request_id = self.request_id,
is_response = self.is_response, is_response = self.is_response,
advertise = False, advertise = False,
auto_compress = self.auto_compress,
) )
def validate_proof(self, proof_data): def validate_proof(self, proof_data):
@ -981,7 +985,7 @@ class Resource:
processed_segments = self.segment_index-1 processed_segments = self.segment_index-1
current_segment_parts = self.total_parts current_segment_parts = self.total_parts
max_parts_per_segment = math.ceil(Resource.MAX_EFFICIENT_SIZE/Resource.SDU) max_parts_per_segment = math.ceil(Resource.MAX_EFFICIENT_SIZE/self.sdu)
previously_processed_parts = processed_segments*max_parts_per_segment previously_processed_parts = processed_segments*max_parts_per_segment
@ -1004,7 +1008,7 @@ class Resource:
processed_segments = self.segment_index-1 processed_segments = self.segment_index-1
current_segment_parts = self.total_parts current_segment_parts = self.total_parts
max_parts_per_segment = math.ceil(Resource.MAX_EFFICIENT_SIZE/Resource.SDU) max_parts_per_segment = math.ceil(Resource.MAX_EFFICIENT_SIZE/self.sdu)
previously_processed_parts = processed_segments*max_parts_per_segment previously_processed_parts = processed_segments*max_parts_per_segment

10
RNS/Reticulum.py
View File

@ -89,6 +89,16 @@ class Reticulum:
the default value. the default value.
""" """
LINK_MTU_DISCOVERY = False
"""
Whether automatic link MTU discovery is enabled by default in this
release. Link MTU discovery significantly increases throughput over
fast links, but requires all intermediary hops to also support it.
Support for this feature was added in RNS version 0.9.0. This option
will become enabled by default in the near future. Please update your
RNS instances.
"""
MAX_QUEUED_ANNOUNCES = 16384 MAX_QUEUED_ANNOUNCES = 16384
QUEUED_ANNOUNCE_LIFE = 60*60*24 QUEUED_ANNOUNCE_LIFE = 60*60*24

9
RNS/Utilities/rncp.py
View File

@ -581,7 +581,7 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
exit(0) exit(0)
def send(configdir, verbosity = 0, quietness = 0, destination = None, file = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT, silent=False, phy_rates=False): def send(configdir, verbosity = 0, quietness = 0, destination = None, file = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT, silent=False, phy_rates=False, no_compress=False):
global current_resource, resource_done, link, speed, show_phy_rates global current_resource, resource_done, link, speed, show_phy_rates
from tempfile import TemporaryFile from tempfile import TemporaryFile
targetloglevel = 3+verbosity-quietness targetloglevel = 3+verbosity-quietness
@ -705,7 +705,10 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
print(f"{erase_str}Advertising file resource ", end=es) print(f"{erase_str}Advertising file resource ", end=es)
link.identify(identity) link.identify(identity)
resource = RNS.Resource(temp_file, link, callback = sender_progress, progress_callback = sender_progress) auto_compress = True
if no_compress:
auto_compress = False
resource = RNS.Resource(temp_file, link, callback = sender_progress, progress_callback = sender_progress, auto_compress = auto_compress)
current_resource = resource current_resource = resource
while resource.status < RNS.Resource.TRANSFERRING: while resource.status < RNS.Resource.TRANSFERRING:
@ -784,6 +787,7 @@ def main():
parser.add_argument('-q', '--quiet', action='count', default=0, help="decrease verbosity") parser.add_argument('-q', '--quiet', action='count', default=0, help="decrease verbosity")
parser.add_argument("-S", '--silent', action='store_true', default=False, help="disable transfer progress output") parser.add_argument("-S", '--silent', action='store_true', default=False, help="disable transfer progress output")
parser.add_argument("-l", '--listen', action='store_true', default=False, help="listen for incoming transfer requests") parser.add_argument("-l", '--listen', action='store_true', default=False, help="listen for incoming transfer requests")
parser.add_argument("-C", '--no-compress', action='store_true', default=False, help="disable automatic compression")
parser.add_argument("-F", '--allow-fetch', action='store_true', default=False, help="allow authenticated clients to fetch files") parser.add_argument("-F", '--allow-fetch', action='store_true', default=False, help="allow authenticated clients to fetch files")
parser.add_argument("-f", '--fetch', action='store_true', default=False, help="fetch file from remote listener instead of sending") parser.add_argument("-f", '--fetch', action='store_true', default=False, help="fetch file from remote listener instead of sending")
parser.add_argument("-j", "--jail", metavar="path", action="store", default=None, help="restrict fetch requests to specified path", type=str) parser.add_argument("-j", "--jail", metavar="path", action="store", default=None, help="restrict fetch requests to specified path", type=str)
@ -842,6 +846,7 @@ def main():
timeout = args.w, timeout = args.w,
silent = args.silent, silent = args.silent,
phy_rates = args.phy_rates, phy_rates = args.phy_rates,
no_compress = args.no_compress,
) )
else: else: