From 459f6b792ff4bed5cb95d3a6476e089d49b81cfd Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 10 Jun 2020 10:58:13 +0200 Subject: [PATCH] Optimised resource transfers, fixed resource transfer regression, removed txdelay from UDPInterface. --- Examples/Filetransfer.py | 2 +- Notes/Wire Format | 6 ++-- RNS/Interfaces/UdpInterface.py | 5 --- RNS/Resource.py | 60 ++++++++++++++++++++++++---------- setup.py | 2 +- 5 files changed, 47 insertions(+), 28 deletions(-) diff --git a/Examples/Filetransfer.py b/Examples/Filetransfer.py index cd07d9a..4cb3cac 100644 --- a/Examples/Filetransfer.py +++ b/Examples/Filetransfer.py @@ -20,7 +20,7 @@ import RNS.vendor.umsgpack as umsgpack APP_NAME = "example_utilitites" # We'll also define a default timeout, in seconds -APP_TIMEOUT = 15.0 +APP_TIMEOUT = 45.0 ########################################################## #### Server Part ######################################### diff --git a/Notes/Wire Format b/Notes/Wire Format index f203cf2..a8f20bc 100644 --- a/Notes/Wire Format +++ b/Notes/Wire Format @@ -5,15 +5,15 @@ Header Types type 1 00 Two byte header, one 10 byte address field type 2 01 Two byte header, two 10 byte address fields type 3 10 Reserved -type 4 11 Reserved for extended header format +type 4 11 Reserved Propagation Types ----------------- broadcast 00 transport 01 -relay 10 -tunnel 11 +reserved 10 +reserved 11 Destination Types diff --git a/RNS/Interfaces/UdpInterface.py b/RNS/Interfaces/UdpInterface.py index d52f39f..3373d82 100755 --- a/RNS/Interfaces/UdpInterface.py +++ b/RNS/Interfaces/UdpInterface.py @@ -11,10 +11,6 @@ class UdpInterface(Interface): def __init__(self, owner, name, bindip=None, bindport=None, forwardip=None, forwardport=None): self.IN = True self.OUT = False - - # TODO: Optimise so this is not needed - self.transmit_delay = 0.001 - self.name = name if (bindip != None and bindport != None): @@ -45,7 +41,6 @@ class UdpInterface(Interface): self.owner.inbound(data, self) def processOutgoing(self,data): - time.sleep(self.transmit_delay) udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) udp_socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) udp_socket.sendto(data, (self.forward_ip, self.forward_port)) diff --git a/RNS/Resource.py b/RNS/Resource.py index 3b33f3c..7ff8d29 100644 --- a/RNS/Resource.py +++ b/RNS/Resource.py @@ -15,13 +15,25 @@ class Resource: SDU = RNS.Packet.MDU RANDOM_HASH_SIZE = 4 + # This is an indication of what the + # maximum size a resource should be, if + # it is to be handled within reasonable + # time constraint, even on small systems. + # + # A small system in this regard is + # defined as a Raspberry Pi, which should + # be able to compress, encrypt and hasmap + # the resource in about 10 seconds. + MAX_EFFICIENT_SIZE = 16 * 1024 * 1024 + # The maximum size to auto-compress with # bz2 before sending. - AUTO_COMPRESS_MAX_SIZE = 32 * 1024 * 1024 + AUTO_COMPRESS_MAX_SIZE = MAX_EFFICIENT_SIZE # TODO: Should be allocated more # intelligently - MAX_RETRIES = 2 + # TODO: Set higher + MAX_RETRIES = 5 SENDER_GRACE_TIME = 10 RETRY_GRACE_TIME = 0.25 @@ -72,6 +84,8 @@ class Resource: resource.hashmap_height = 0 resource.waiting_for_hmu = False + resource.receiving_part = False + resource.consecutive_completed_height = 0 resource.link.register_incoming_resource(resource) @@ -344,14 +358,10 @@ class Resource: sleep(sleep_time) def assemble(self): - # TODO: Optimise assembly. It's way too - # slow for larger files if not self.status == Resource.FAILED: try: self.status = Resource.ASSEMBLING - stream = b"" - for part in self.parts: - stream += part + stream = b"".join(self.parts) if self.encrypted: data = self.link.decrypt(stream) @@ -412,6 +422,10 @@ class Resource: def receive_part(self, packet): + while self.receiving_part: + sleep(0.001) + + self.receiving_part = True self.last_activity = time.time() self.retries_left = self.max_retries @@ -439,21 +453,25 @@ class Resource: self.outstanding_parts -= 1 # Update consecutive completed pointer - if i == 0 or self.parts[i-1] != None and i == self.consecutive_completed_height: - self.consecutive_completed_height = i+1 - cp = i+1 - while cp < len(self.parts) and self.parts[cp] != None: - self.consecutive_completed_height = cp - cp += 1 + if i == self.consecutive_completed_height + 1: + self.consecutive_completed_height = i + + cp = self.consecutive_completed_height + 1 + while cp < len(self.parts) and self.parts[cp] != None: + self.consecutive_completed_height = cp + cp += 1 + i += 1 + self.receiving_part = False + if self.__progress_callback != None: self.__progress_callback(self) if self.outstanding_parts == 0 and self.received_count == self.total_parts: self.assemble() elif self.outstanding_parts == 0: - # TODO: Figure out if ther is a mathematically + # TODO: Figure out if there is a mathematically # optimal way to adjust windows if self.window < self.window_max: self.window += 1 @@ -461,18 +479,25 @@ class Resource: self.window_min += 1 self.request_next() + else: + self.receiving_part = False # Called on incoming resource to send a request for more data def request_next(self): + while self.receiving_part: + sleep(0.001) + if not self.status == Resource.FAILED: if not self.waiting_for_hmu: self.outstanding_parts = 0 hashmap_exhausted = Resource.HASHMAP_IS_NOT_EXHAUSTED requested_hashes = b"" - i = 0; pn = self.consecutive_completed_height - for part in self.parts[self.consecutive_completed_height:self.consecutive_completed_height+self.window]: - + offset = (1 if self.consecutive_completed_height > 0 else 0) + i = 0; pn = self.consecutive_completed_height+offset + search_start = pn + + for part in self.parts[search_start:search_start+self.window]: if part == None: part_hash = self.hashmap[pn] if part_hash != None: @@ -622,7 +647,6 @@ class Resource: class ResourceAdvertisement: - # TODO: Can this be allocated dynamically? Keep in mind hashmap_update inference HASHMAP_MAX_LEN = 84 COLLISION_GUARD_SIZE = 2*Resource.WINDOW_MAX+HASHMAP_MAX_LEN diff --git a/setup.py b/setup.py index 31b041f..357a8f7 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="rns", - version="0.1.5", + version="0.1.6", author="Mark Qvist", author_email="mark@unsigned.io", description="Self-configuring, encrypted and resilient mesh networking stack for LoRa, packet radio, WiFi and everything in between",