Merge 86d4028dc4 into 527f6cc906

RNS/Interfaces/LocalInterface.py

@@ -176,8 +176,8 @@ class LocalClientInterface(Interface):
                         self.send_lock = Lock()
 
                     with self.send_lock:
-                        # RNS.log(f"Simulating latency of {RNS.prettytime(s)} for {len(data)} bytes", RNS.LOG_EXTREME)
                         s = len(data) / self.bitrate * 8
+                        RNS.log(f"Simulating latency of {RNS.prettytime(s)} for {len(data)} bytes")
                         time.sleep(s)
 
                 data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])

RNS/Resource.py

@@ -160,7 +160,7 @@ class Resource:
             resource.encrypted           = True if resource.flags & 0x01 else False
             resource.compressed          = True if resource.flags >> 1 & 0x01 else False
             resource.initiator           = False
-            resource.callback            = callback
+            resource.callback             = callback
             resource.__progress_callback = progress_callback
             resource.total_parts         = int(math.ceil(resource.size/float(Resource.SDU)))
             resource.received_count      = 0
@@ -233,6 +233,7 @@ class Resource:
                 data_size = os.stat(data.name).st_size
 
             self.total_size = data_size
+            self.grand_total_parts = math.ceil(data_size/Resource.SDU)
 
             if data_size <= Resource.MAX_EFFICIENT_SIZE:
                 self.total_segments = 1
@@ -253,6 +254,7 @@ class Resource:
 
         elif isinstance(data, bytes):
             data_size = len(data)
+            self.grand_total_parts = math.ceil(data_size/Resource.SDU)
             self.total_size  = data_size
 
             resource_data = data
@@ -346,7 +348,6 @@ class Resource:
             self.size = len(self.data)
             self.sent_parts = 0
             hashmap_entries = int(math.ceil(self.size/float(Resource.SDU)))
-            self.total_parts = hashmap_entries
 
             hashmap_ok = False
             while not hashmap_ok:
@@ -962,68 +963,21 @@ class Resource:
         """
         if self.status == RNS.Resource.COMPLETE and self.segment_index == self.total_segments:
             return 1.0
-
         elif self.initiator:
-            if not self.split:
+            self.processed_parts  = (self.segment_index-1)*math.ceil(Resource.MAX_EFFICIENT_SIZE/Resource.SDU)
-                self.processed_parts = self.sent_parts
+            self.processed_parts += self.sent_parts
-                self.progress_total_parts = float(self.total_parts)
+            self.progress_total_parts = float(self.grand_total_parts)
-
-            else:
-                is_last_segment = self.segment_index != self.total_segments
-                total_segments = self.total_segments
-                processed_segments = self.segment_index-1
-
-                current_segment_parts = self.total_parts
-                max_parts_per_segment = math.ceil(Resource.MAX_EFFICIENT_SIZE/Resource.SDU)
-
-                previously_processed_parts = processed_segments*max_parts_per_segment
-
-                if current_segment_parts < max_parts_per_segment:
-                    current_segment_factor = max_parts_per_segment / current_segment_parts
-                else:
-                    current_segment_factor = 1
-
-                self.processed_parts = previously_processed_parts + self.sent_parts*current_segment_factor
-                self.progress_total_parts = self.total_segments*max_parts_per_segment
-
         else:
-            if not self.split:
+            self.processed_parts  = (self.segment_index-1)*math.ceil(Resource.MAX_EFFICIENT_SIZE/Resource.SDU)
-                self.processed_parts = self.received_count
+            self.processed_parts += self.received_count
-                self.progress_total_parts = float(self.total_parts)
+            if self.split:
-
+                self.progress_total_parts = float(math.ceil(self.total_size/Resource.SDU))
             else:
-                is_last_segment = self.segment_index != self.total_segments
+                self.progress_total_parts = float(self.total_parts)
-                total_segments = self.total_segments
-                processed_segments = self.segment_index-1
-
-                current_segment_parts = self.total_parts
-                max_parts_per_segment = math.ceil(Resource.MAX_EFFICIENT_SIZE/Resource.SDU)
-
-                previously_processed_parts = processed_segments*max_parts_per_segment
-
-                if current_segment_parts < max_parts_per_segment:
-                    current_segment_factor = max_parts_per_segment / current_segment_parts
-                else:
-                    current_segment_factor = 1
-
-                self.processed_parts = previously_processed_parts + self.received_count*current_segment_factor
-                self.progress_total_parts = self.total_segments*max_parts_per_segment
-
 
         progress = min(1.0, self.processed_parts / self.progress_total_parts)
         return progress
 
-    def get_segment_progress(self):
-        if self.status == RNS.Resource.COMPLETE and self.segment_index == self.total_segments:
-            return 1.0
-        elif self.initiator:
-            processed_parts = self.sent_parts
-        else:
-            processed_parts = self.received_count
-
-        progress = min(1.0, processed_parts / self.total_parts)
-        return progress
-
     def get_transfer_size(self):
         """
         :returns: The number of bytes needed to transfer the resource.

RNS/Utilities/rncp.py

@@ -35,14 +35,10 @@ APP_NAME = "rncp"
 allow_all = False
 allow_fetch = False
 fetch_jail = None
-show_phy_rates = False
 allowed_identity_hashes = []
 
 REQ_FETCH_NOT_ALLOWED = 0xF0
 
-es = " "
-erase_str = "\33[2K\r"
-
 def listen(configdir, verbosity = 0, quietness = 0, allowed = [], display_identity = False,
            limit = None, disable_auth = None, fetch_allowed = False, jail = None, announce = False):
     global allow_all, allow_fetch, allowed_identity_hashes, fetch_jail
@@ -134,13 +130,10 @@ def listen(configdir, verbosity = 0, quietness = 0, allowed = [], display_identi
         if not allow_fetch:
             return REQ_FETCH_NOT_ALLOWED
 
+        file_path = os.path.abspath(os.path.expanduser(data))
         if fetch_jail:
-            file_path = os.path.abspath(os.path.expanduser(f"{fetch_jail}/{data}"))
+            if not file_path.startswith(jail):
-            RNS.log(f"Disallowing fetch request for {file_path} outside of fetch jail {fetch_jail}", RNS.LOG_WARNING)
-            if not file_path.startswith(fetch_jail):
                 return REQ_FETCH_NOT_ALLOWED
-        else:
-            file_path = os.path.abspath(os.path.expanduser(f"{data}"))
 
         target_link = None
         for link in RNS.Transport.active_links:
@@ -175,13 +168,7 @@ def listen(configdir, verbosity = 0, quietness = 0, allowed = [], display_identi
 
 
     destination.set_link_established_callback(client_link_established)
-    if allow_fetch:
+    destination.register_request_handler("fetch_file", response_generator=fetch_request, allow=RNS.Destination.ALLOW_LIST, allowed_list=allowed_identity_hashes)
-        if allow_all:
-            RNS.log("Allowing unauthenticated fetch requests", RNS.LOG_WARNING)
-            destination.register_request_handler("fetch_file", response_generator=fetch_request, allow=RNS.Destination.ALLOW_ALL)
-        else:
-            destination.register_request_handler("fetch_file", response_generator=fetch_request, allow=RNS.Destination.ALLOW_LIST, allowed_list=allowed_identity_hashes)
-
     print(f"rncp listening on {RNS.prettyhexrep(destination.hash)}")
 
     if announce >= 0:
@@ -240,7 +227,6 @@ def receive_resource_started(resource):
     print(f"Starting resource transfer {RNS.prettyhexrep(resource.hash)}{id_str}")
 
 def receive_resource_concluded(resource):
-    global fetch_jail
     if resource.status == RNS.Resource.COMPLETE:
         print(f"{resource} completed")
 
@@ -249,20 +235,12 @@ def receive_resource_concluded(resource):
             filename = resource.data.read(filename_len).decode("utf-8")
 
             counter = 0
-            if fetch_jail:
+            saved_filename = filename
-                saved_filename = os.path.abspath(os.path.expanduser(fetch_jail+"/"+filename))
+            while os.path.isfile(saved_filename):
-                if not saved_filename.startswith(fetch_jail):
-                    RNS.log(f"Invalid save path {saved_filename}, ignoring", RNS.LOG_ERROR)
-                    return
-            else:
-                saved_filename = filename
-
-            full_save_path = saved_filename
-            while os.path.isfile(full_save_path):
                 counter += 1
-                full_save_path = f"{saved_filename}.{counter}"
+                saved_filename = f"{filename}.{counter}"
 
-            file = open(full_save_path, "wb")
+            file = open(saved_filename, "wb")
             file.write(resource.data.read())
             file.close()
 
@@ -276,45 +254,33 @@ resource_done = False
 current_resource = None
 stats = []
 speed = 0.0
-phy_speed = 0.0
 def sender_progress(resource):
     stats_max = 32
-    global current_resource, stats, speed, phy_speed, resource_done
+    global current_resource, stats, speed, resource_done
     current_resource = resource
-    
     now = time.time()
-    got = current_resource.get_progress()*current_resource.get_data_size()
+    got = current_resource.get_progress()*current_resource.total_size
-    phy_got = current_resource.get_segment_progress()*current_resource.get_transfer_size()
+    entry = [now, got]
-    
-    entry = [now, got, phy_got]
     stats.append(entry)
-
     while len(stats) > stats_max:
         stats.pop(0)
 
     span = now - stats[0][0]
     if span == 0:
         speed = 0
-        phy_speed = 0
-    
     else:
         diff = got - stats[0][1]
         speed = diff/span
 
-        phy_diff = phy_got - stats[0][2]
-        if phy_diff > 0:
-            phy_speed = phy_diff/span
-
     if resource.status < RNS.Resource.COMPLETE:
         resource_done = False
     else:
         resource_done = True
 
 link = None
-def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT, silent=False, phy_rates=False):
+def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT, silent=False):
-    global current_resource, resource_done, link, speed, show_phy_rates
+    global current_resource, resource_done, link, speed
     targetloglevel = 3+verbosity-quietness
-    show_phy_rates = phy_rates
 
     try:
         dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
@@ -349,7 +315,7 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
         if silent:
             print(f"Path to {RNS.prettyhexrep(destination_hash)} requested")
         else:
-            print(f"Path to {RNS.prettyhexrep(destination_hash)} requested  ", end=es)
+            print(f"Path to {RNS.prettyhexrep(destination_hash)} requested  ", end=" ")
         sys.stdout.flush()
 
     i = 0
@@ -366,13 +332,13 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
         if silent:
             print("Path not found")
         else:
-            print(f"{erase_str}Path not found")
+            print("\r                                                            \rPath not found")
         exit(1)
     else:
         if silent:
             print(f"Establishing link with {RNS.prettyhexrep(destination_hash)}")
         else:
-            print(f"{erase_str}Establishing link with {RNS.prettyhexrep(destination_hash)}  ", end=es)
+            print(f'\r                                                            \rEstablishing link with {RNS.prettyhexrep(destination_hash)}  ', end=" ")
 
     listener_identity = RNS.Identity.recall(destination_hash)
     listener_destination = RNS.Destination(
@@ -395,13 +361,13 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
         if silent:
             print(f"Could not establish link with {RNS.prettyhexrep(destination_hash)}")
         else:
-            print(f"{erase_str}Could not establish link with {RNS.prettyhexrep(destination_hash)}")
+            print(f'\r                                                            \rCould not establish link with {RNS.prettyhexrep(destination_hash)}')
         exit(1)
     else:
         if silent:
             print("Requesting file from remote...")
         else:
-            print(f"{erase_str}Requesting file from remote  ", end=es)
+            print("\r                                                            \rRequesting file from remote  ", end=" ")
 
     link.identify(identity)
 
@@ -476,31 +442,31 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
             i = (i+1)%len(syms)
 
     if request_status == "fetch_not_allowed":
-        if not silent: print(f"{erase_str}", end="")
+        if not silent: print("\r                                                            \r", end="")
         print(f"Fetch request failed, fetching the file {file} was not allowed by the remote")
         link.teardown()
         time.sleep(0.15)
         exit(0)
     elif request_status == "not_found":
-        if not silent: print(f"{erase_str}", end="")
+        if not silent: print("\r                                                            \r", end="")
         print(f"Fetch request failed, the file {file} was not found on the remote")
         link.teardown()
         time.sleep(0.15)
         exit(0)
     elif request_status == "remote_error":
-        if not silent: print(f"{erase_str}", end="")
+        if not silent: print("\r                                                            \r", end="")
         print("Fetch request failed due to an error on the remote system")
         link.teardown()
         time.sleep(0.15)
         exit(0)
     elif request_status == "unknown":
-        if not silent: print(f"{erase_str}", end="")
+        if not silent: print("\r                                                            \r", end="")
         print("Fetch request failed due to an unknown error (probably not authorised)")
         link.teardown()
         time.sleep(0.15)
         exit(0)
     elif request_status == "found":
-        if not silent: print(f"{erase_str}", end="")
+        if not silent: print("\r                                                            \r", end="")
 
     while not resource_resolved:
         if not silent:
@@ -508,21 +474,13 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
             if current_resource:
                 prg = current_resource.get_progress()
                 percent = round(prg * 100.0, 1)
-                if show_phy_rates:
+                stat_str = f"{percent}% - {size_str(int(prg * current_resource.total_size))} of {size_str(current_resource.total_size)} - {size_str(speed, 'b')}ps"
-                    pss = size_str(phy_speed, "b")
-                    phy_str = f" ({pss}ps at physical layer)"
-                else:
-                    phy_str = ""
-                ps = size_str(int(prg*current_resource.total_size))
-                ts = size_str(current_resource.total_size)
-                ss = size_str(speed, "b")
-                stat_str = f"{percent}% - {ps} of {ts} - {ss}ps{phy_str}"
                 if prg != 1.0:
-                    print(f"{erase_str}Transferring file {syms[i]} {stat_str}", end=es)
+                    print(f'\r                                                                                  \rTransferring file {syms[i]} {stat_str}', end=" ")
                 else:
-                    print(f"{erase_str}Transfer complete  {stat_str}", end=es)
+                    print(f'\r                                                                                  \rTransfer complete  {stat_str}', end=" ")
             else:
-                print(f"{erase_str}Waiting for transfer to start {syms[i]} ", end=es)
+                print(f'\r                                                                                  \rWaiting for transfer to start {syms[i]} ', end=" ")
             sys.stdout.flush()
             i = (i+1)%len(syms)
 
@@ -530,12 +488,13 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
         if silent:
             print("The transfer failed")
         else:
-            print(f"{erase_str}The transfer failed")
+            print("\r                                                            \rThe transfer failed")
         exit(1)
     else:
         if silent:
             print(f"{file} fetched from {RNS.prettyhexrep(destination_hash)}")
         else:
+            #print("\r                                                                                  \r"+str(file)+" fetched from "+RNS.prettyhexrep(destination_hash))
             print(f"\n{file} fetched from {RNS.prettyhexrep(destination_hash)}")
         link.teardown()
         time.sleep(0.15)
@@ -545,11 +504,10 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
     exit(0)
 
 
-def send(configdir, verbosity = 0, quietness = 0, destination = None, file = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT, silent=False, phy_rates=False):
+def send(configdir, verbosity = 0, quietness = 0, destination = None, file = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT, silent=False):
-    global current_resource, resource_done, link, speed, show_phy_rates
+    global current_resource, resource_done, link, speed
     from tempfile import TemporaryFile
     targetloglevel = 3+verbosity-quietness
-    show_phy_rates = phy_rates
 
     try:
         dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
@@ -578,14 +536,14 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         print("Filename exceeds max size, cannot send")
         exit(1)
     else:
-        print("Preparing file...", end=es)
+        print("Preparing file...", end=" ")
 
     temp_file.write(filename_len.to_bytes(2, "big"))
     temp_file.write(filename_bytes)
     temp_file.write(real_file.read())
     temp_file.seek(0)
 
-    print(f"{erase_str}", end="")
+    print("\r                                                            \r", end="")
 
     reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel)
 
@@ -608,7 +566,7 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         if silent:
             print(f"Path to {RNS.prettyhexrep(destination_hash)} requested")
         else:
-            print(f"Path to {RNS.prettyhexrep(destination_hash)} requested  ", end=es)
+            print(f"Path to {RNS.prettyhexrep(destination_hash)} requested  ", end=" ")
         sys.stdout.flush()
 
     i = 0
@@ -625,13 +583,13 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         if silent:
             print("Path not found")
         else:
-            print(f"{erase_str}Path not found")
+            print("\r                                                            \rPath not found")
         exit(1)
     else:
         if silent:
             print(f"Establishing link with {RNS.prettyhexrep(destination_hash)}")
         else:
-            print(f"{erase_str}Establishing link with {RNS.prettyhexrep(destination_hash)} ", end=es)
+            print(f'\r                                                            \rEstablishing link with {RNS.prettyhexrep(destination_hash)} ', end=" ")
 
     receiver_identity = RNS.Identity.recall(destination_hash)
     receiver_destination = RNS.Destination(
@@ -654,19 +612,19 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         if silent:
             print(f"Link establishment with {RNS.prettyhexrep(destination_hash)} timed out")
         else:
-            print(f"{erase_str}Link establishment with {RNS.prettyhexrep(destination_hash)} timed out")
+            print(f'\r                                                            \rLink establishment with {RNS.prettyhexrep(destination_hash)} timed out')
         exit(1)
     elif not RNS.Transport.has_path(destination_hash):
         if silent:
             print(f"No path found to {RNS.prettyhexrep(destination_hash)}")
         else:
-            print(f"{erase_str}No path found to {RNS.prettyhexrep(destination_hash)}")
+            print(f'\r                                                            \rNo path found to {RNS.prettyhexrep(destination_hash)}')
         exit(1)
     else:
         if silent:
             print("Advertising file resource...")
         else:
-            print(f"{erase_str}Advertising file resource  ", end=es)
+            print("\r                                                            \rAdvertising file resource  ", end=" ")
 
     link.identify(identity)
     resource = RNS.Resource(temp_file, link, callback = sender_progress, progress_callback = sender_progress)
@@ -684,32 +642,23 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         if silent:
             print(f"File was not accepted by {RNS.prettyhexrep(destination_hash)}")
         else:
-            print(f"{erase_str}File was not accepted by {RNS.prettyhexrep(destination_hash)}")
+            print(f'\r                                                            \rFile was not accepted by {RNS.prettyhexrep(destination_hash)}')
         exit(1)
     else:
         if silent:
             print("Transferring file...")
         else:
-            print(f"{erase_str}Transferring file  ", end=es)
+            print("\r                                                            \rTransferring file  ", end=" ")
 
     def progress_update(i, done=False):
         time.sleep(0.1)
         prg = current_resource.get_progress()
         percent = round(prg * 100.0, 1)
-        if show_phy_rates:
+        stat_str = f"{percent}% - {size_str(int(prg * current_resource.total_size))} of {size_str(current_resource.total_size)} - {size_str(speed, 'b')}ps"
-            pss = size_str(phy_speed, "b")
-            phy_str = f" ({pss}ps at physical layer)"
-        else:
-            phy_str = ""
-        es = "  "
-        cs = size_str(int(prg*current_resource.total_size))
-        ts = size_str(current_resource.total_size)
-        ss = size_str(speed, "b")
-        stat_str = f"{percent}% - {cs} of {ts} - {ss}ps{phy_str}"
         if not done:
-            print(f"{erase_str}Transferring file {syms[i]} {stat_str}", end=es)
+            print(f'\r                                                                                  \rTransferring file {syms[i]} {stat_str}', end=" ")
         else:
-            print(f"{erase_str}Transfer complete  {stat_str}", end=es)
+            print(f'\r                                                                                  \rTransfer complete  {stat_str}', end=" ")
         sys.stdout.flush()
         i = (i+1)%len(syms)
         return i
@@ -725,12 +674,13 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         if silent:
             print("The transfer failed")
         else:
-            print(f"{erase_str}The transfer failed")
+            print("\r                                                            \rThe transfer failed")
         exit(1)
     else:
         if silent:
             print(f"{file_path} copied to {RNS.prettyhexrep(destination_hash)}")
         else:
+            # print("\r                                                                                  \r"+str(file_path)+" copied to "+RNS.prettyhexrep(destination_hash))
             print(f"\n{file_path} copied to {RNS.prettyhexrep(destination_hash)}")
         link.teardown()
         time.sleep(0.25)
@@ -756,7 +706,6 @@ def main():
         parser.add_argument('-n', '--no-auth', action='store_true', default=False, help="accept requests from anyone")
         parser.add_argument('-p', '--print-identity', action='store_true', default=False, help="print identity and destination info and exit")
         parser.add_argument("-w", action="store", metavar="seconds", type=float, help="sender timeout before giving up", default=RNS.Transport.PATH_REQUEST_TIMEOUT)
-        parser.add_argument('-P', '--phy-rates', action='store_true', default=False, help="display physical layer transfer rates")
         # parser.add_argument("--limit", action="store", metavar="files", type=float, help="maximum number of files to accept", default=None)
         parser.add_argument("--version", action="version", version=f"rncp {__version__}")
 
@@ -786,7 +735,6 @@ def main():
                     file = args.file,
                     timeout = args.w,
                     silent = args.silent,
-                    phy_rates = args.phy_rates,
                 )
             else:
                 print("")
@@ -802,7 +750,6 @@ def main():
                 file = args.file,
                 timeout = args.w,
                 silent = args.silent,
-                phy_rates = args.phy_rates,
             )
 
         else:

RNS/_version.py

@@ -1 +1 @@
-__version__ = "0.8.3"
+__version__ = "0.8.2"