Merge f82ee02438 into 715f4d9fcb

RNS/Interfaces/LocalInterface.py

@@ -176,8 +176,8 @@ class LocalClientInterface(Interface):
                         self.send_lock = Lock()
 
                     with self.send_lock:
+                        # RNS.log(f"Simulating latency of {RNS.prettytime(s)} for {len(data)} bytes", RNS.LOG_EXTREME)
                         s = len(data) / self.bitrate * 8
-                        RNS.log(f"Simulating latency of {RNS.prettytime(s)} for {len(data)} bytes")
                         time.sleep(s)
 
                 data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])

RNS/Resource.py

@@ -160,7 +160,7 @@ class Resource:
             resource.encrypted           = True if resource.flags & 0x01 else False
             resource.compressed          = True if resource.flags >> 1 & 0x01 else False
             resource.initiator           = False
-            resource.callback             = callback
+            resource.callback            = callback
             resource.__progress_callback = progress_callback
             resource.total_parts         = int(math.ceil(resource.size/float(Resource.SDU)))
             resource.received_count      = 0
@@ -233,7 +233,6 @@ class Resource:
                 data_size = os.stat(data.name).st_size
 
             self.total_size = data_size
-            self.grand_total_parts = math.ceil(data_size/Resource.SDU)
 
             if data_size <= Resource.MAX_EFFICIENT_SIZE:
                 self.total_segments = 1
@@ -254,7 +253,6 @@ class Resource:
 
         elif isinstance(data, bytes):
             data_size = len(data)
-            self.grand_total_parts = math.ceil(data_size/Resource.SDU)
             self.total_size  = data_size
 
             resource_data = data
@@ -348,6 +346,7 @@ class Resource:
             self.size = len(self.data)
             self.sent_parts = 0
             hashmap_entries = int(math.ceil(self.size/float(Resource.SDU)))
+            self.total_parts = hashmap_entries
 
             hashmap_ok = False
             while not hashmap_ok:
@@ -963,21 +962,68 @@ class Resource:
         """
         if self.status == RNS.Resource.COMPLETE and self.segment_index == self.total_segments:
             return 1.0
+
         elif self.initiator:
-            self.processed_parts  = (self.segment_index-1)*math.ceil(Resource.MAX_EFFICIENT_SIZE/Resource.SDU)
+            if not self.split:
-            self.processed_parts += self.sent_parts
+                self.processed_parts = self.sent_parts
-            self.progress_total_parts = float(self.grand_total_parts)
-        else:
-            self.processed_parts  = (self.segment_index-1)*math.ceil(Resource.MAX_EFFICIENT_SIZE/Resource.SDU)
-            self.processed_parts += self.received_count
-            if self.split:
-                self.progress_total_parts = float(math.ceil(self.total_size/Resource.SDU))
-            else:
                 self.progress_total_parts = float(self.total_parts)
 
+            else:
+                is_last_segment = self.segment_index != self.total_segments
+                total_segments = self.total_segments
+                processed_segments = self.segment_index-1
+
+                current_segment_parts = self.total_parts
+                max_parts_per_segment = math.ceil(Resource.MAX_EFFICIENT_SIZE/Resource.SDU)
+
+                previously_processed_parts = processed_segments*max_parts_per_segment
+
+                if current_segment_parts < max_parts_per_segment:
+                    current_segment_factor = max_parts_per_segment / current_segment_parts
+                else:
+                    current_segment_factor = 1
+
+                self.processed_parts = previously_processed_parts + self.sent_parts*current_segment_factor
+                self.progress_total_parts = self.total_segments*max_parts_per_segment
+
+        else:
+            if not self.split:
+                self.processed_parts = self.received_count
+                self.progress_total_parts = float(self.total_parts)
+
+            else:
+                is_last_segment = self.segment_index != self.total_segments
+                total_segments = self.total_segments
+                processed_segments = self.segment_index-1
+
+                current_segment_parts = self.total_parts
+                max_parts_per_segment = math.ceil(Resource.MAX_EFFICIENT_SIZE/Resource.SDU)
+
+                previously_processed_parts = processed_segments*max_parts_per_segment
+
+                if current_segment_parts < max_parts_per_segment:
+                    current_segment_factor = max_parts_per_segment / current_segment_parts
+                else:
+                    current_segment_factor = 1
+
+                self.processed_parts = previously_processed_parts + self.received_count*current_segment_factor
+                self.progress_total_parts = self.total_segments*max_parts_per_segment
+
+
         progress = min(1.0, self.processed_parts / self.progress_total_parts)
         return progress
 
+    def get_segment_progress(self):
+        if self.status == RNS.Resource.COMPLETE and self.segment_index == self.total_segments:
+            return 1.0
+        elif self.initiator:
+            processed_parts = self.sent_parts
+        else:
+            processed_parts = self.received_count
+
+        progress = min(1.0, processed_parts / self.total_parts)
+        return progress
+
     def get_transfer_size(self):
         """
         :returns: The number of bytes needed to transfer the resource.
@@ -1160,4 +1206,4 @@ class ResourceAdvertisement:
         adv.u = True if ((adv.f >> 3) & 0x01) == 0x01 else False
         adv.p = True if ((adv.f >> 4) & 0x01) == 0x01 else False
 
-        return adv
+        return adv

RNS/Utilities/rncp.py

@@ -35,10 +35,14 @@ APP_NAME = "rncp"
 allow_all = False
 allow_fetch = False
 fetch_jail = None
+show_phy_rates = False
 allowed_identity_hashes = []
 
 REQ_FETCH_NOT_ALLOWED = 0xF0
 
+es = " "
+erase_str = "\33[2K\r"
+
 def listen(configdir, verbosity = 0, quietness = 0, allowed = [], display_identity = False,
            limit = None, disable_auth = None, fetch_allowed = False, jail = None, announce = False):
     global allow_all, allow_fetch, allowed_identity_hashes, fetch_jail
@@ -130,10 +134,13 @@ def listen(configdir, verbosity = 0, quietness = 0, allowed = [], display_identi
         if not allow_fetch:
             return REQ_FETCH_NOT_ALLOWED
 
-        file_path = os.path.abspath(os.path.expanduser(data))
         if fetch_jail:
-            if not file_path.startswith(jail):
+            file_path = os.path.abspath(os.path.expanduser(f"{fetch_jail}/{data}"))
+            RNS.log(f"Disallowing fetch request for {file_path} outside of fetch jail {fetch_jail}", RNS.LOG_WARNING)
+            if not file_path.startswith(fetch_jail):
                 return REQ_FETCH_NOT_ALLOWED
+        else:
+            file_path = os.path.abspath(os.path.expanduser(f"{data}"))
 
         target_link = None
         for link in RNS.Transport.active_links:
@@ -168,7 +175,13 @@ def listen(configdir, verbosity = 0, quietness = 0, allowed = [], display_identi
 
 
     destination.set_link_established_callback(client_link_established)
-    destination.register_request_handler("fetch_file", response_generator=fetch_request, allow=RNS.Destination.ALLOW_LIST, allowed_list=allowed_identity_hashes)
+    if allow_fetch:
+        if allow_all:
+            RNS.log("Allowing unauthenticated fetch requests", RNS.LOG_WARNING)
+            destination.register_request_handler("fetch_file", response_generator=fetch_request, allow=RNS.Destination.ALLOW_ALL)
+        else:
+            destination.register_request_handler("fetch_file", response_generator=fetch_request, allow=RNS.Destination.ALLOW_LIST, allowed_list=allowed_identity_hashes)
+
     print(f"rncp listening on {RNS.prettyhexrep(destination.hash)}")
 
     if announce >= 0:
@@ -227,6 +240,7 @@ def receive_resource_started(resource):
     print(f"Starting resource transfer {RNS.prettyhexrep(resource.hash)}{id_str}")
 
 def receive_resource_concluded(resource):
+    global fetch_jail
     if resource.status == RNS.Resource.COMPLETE:
         print(f"{resource} completed")
 
@@ -235,12 +249,20 @@ def receive_resource_concluded(resource):
             filename = resource.data.read(filename_len).decode("utf-8")
 
             counter = 0
-            saved_filename = filename
+            if fetch_jail:
-            while os.path.isfile(saved_filename):
+                saved_filename = os.path.abspath(os.path.expanduser(fetch_jail+"/"+filename))
-                counter += 1
+                if not saved_filename.startswith(fetch_jail):
-                saved_filename = f"{filename}.{counter}"
+                    RNS.log(f"Invalid save path {saved_filename}, ignoring", RNS.LOG_ERROR)
+                    return
+            else:
+                saved_filename = filename
 
-            file = open(saved_filename, "wb")
+            full_save_path = saved_filename
+            while os.path.isfile(full_save_path):
+                counter += 1
+                full_save_path = f"{saved_filename}.{counter}"
+
+            file = open(full_save_path, "wb")
             file.write(resource.data.read())
             file.close()
 
@@ -254,33 +276,45 @@ resource_done = False
 current_resource = None
 stats = []
 speed = 0.0
+phy_speed = 0.0
 def sender_progress(resource):
     stats_max = 32
-    global current_resource, stats, speed, resource_done
+    global current_resource, stats, speed, phy_speed, resource_done
     current_resource = resource
+    
     now = time.time()
-    got = current_resource.get_progress()*current_resource.total_size
+    got = current_resource.get_progress()*current_resource.get_data_size()
-    entry = [now, got]
+    phy_got = current_resource.get_segment_progress()*current_resource.get_transfer_size()
+    
+    entry = [now, got, phy_got]
     stats.append(entry)
+
     while len(stats) > stats_max:
         stats.pop(0)
 
     span = now - stats[0][0]
     if span == 0:
         speed = 0
+        phy_speed = 0
+    
     else:
         diff = got - stats[0][1]
         speed = diff/span
 
+        phy_diff = phy_got - stats[0][2]
+        if phy_diff > 0:
+            phy_speed = phy_diff/span
+
     if resource.status < RNS.Resource.COMPLETE:
         resource_done = False
     else:
         resource_done = True
 
 link = None
-def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT, silent=False):
+def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT, silent=False, phy_rates=False):
-    global current_resource, resource_done, link, speed
+    global current_resource, resource_done, link, speed, show_phy_rates
     targetloglevel = 3+verbosity-quietness
+    show_phy_rates = phy_rates
 
     try:
         dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
@@ -315,7 +349,7 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
         if silent:
             print(f"Path to {RNS.prettyhexrep(destination_hash)} requested")
         else:
-            print(f"Path to {RNS.prettyhexrep(destination_hash)} requested  ", end=" ")
+            print(f"Path to {RNS.prettyhexrep(destination_hash)} requested  ", end=es)
         sys.stdout.flush()
 
     i = 0
@@ -332,13 +366,13 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
         if silent:
             print("Path not found")
         else:
-            print("\r                                                            \rPath not found")
+            print(f"{erase_str}Path not found")
         exit(1)
     else:
         if silent:
             print(f"Establishing link with {RNS.prettyhexrep(destination_hash)}")
         else:
-            print(f'\r                                                            \rEstablishing link with {RNS.prettyhexrep(destination_hash)}  ', end=" ")
+            print(f"{erase_str}Establishing link with {RNS.prettyhexrep(destination_hash)}  ", end=es)
 
     listener_identity = RNS.Identity.recall(destination_hash)
     listener_destination = RNS.Destination(
@@ -361,13 +395,13 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
         if silent:
             print(f"Could not establish link with {RNS.prettyhexrep(destination_hash)}")
         else:
-            print(f'\r                                                            \rCould not establish link with {RNS.prettyhexrep(destination_hash)}')
+            print(f"{erase_str}Could not establish link with {RNS.prettyhexrep(destination_hash)}")
         exit(1)
     else:
         if silent:
             print("Requesting file from remote...")
         else:
-            print("\r                                                            \rRequesting file from remote  ", end=" ")
+            print(f"{erase_str}Requesting file from remote  ", end=es)
 
     link.identify(identity)
 
@@ -442,31 +476,31 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
             i = (i+1)%len(syms)
 
     if request_status == "fetch_not_allowed":
-        if not silent: print("\r                                                            \r", end="")
+        if not silent: print(f"{erase_str}", end="")
         print(f"Fetch request failed, fetching the file {file} was not allowed by the remote")
         link.teardown()
         time.sleep(0.15)
         exit(0)
     elif request_status == "not_found":
-        if not silent: print("\r                                                            \r", end="")
+        if not silent: print(f"{erase_str}", end="")
         print(f"Fetch request failed, the file {file} was not found on the remote")
         link.teardown()
         time.sleep(0.15)
         exit(0)
     elif request_status == "remote_error":
-        if not silent: print("\r                                                            \r", end="")
+        if not silent: print(f"{erase_str}", end="")
         print("Fetch request failed due to an error on the remote system")
         link.teardown()
         time.sleep(0.15)
         exit(0)
     elif request_status == "unknown":
-        if not silent: print("\r                                                            \r", end="")
+        if not silent: print(f"{erase_str}", end="")
         print("Fetch request failed due to an unknown error (probably not authorised)")
         link.teardown()
         time.sleep(0.15)
         exit(0)
     elif request_status == "found":
-        if not silent: print("\r                                                            \r", end="")
+        if not silent: print(f"{erase_str}", end="")
 
     while not resource_resolved:
         if not silent:
@@ -474,13 +508,21 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
             if current_resource:
                 prg = current_resource.get_progress()
                 percent = round(prg * 100.0, 1)
-                stat_str = f"{percent}% - {size_str(int(prg * current_resource.total_size))} of {size_str(current_resource.total_size)} - {size_str(speed, 'b')}ps"
+                if show_phy_rates:
-                if prg != 1.0:
+                    pss = size_str(phy_speed, "b")
-                    print(f'\r                                                                                  \rTransferring file {syms[i]} {stat_str}', end=" ")
+                    phy_str = f" ({pss}ps at physical layer)"
                 else:
-                    print(f'\r                                                                                  \rTransfer complete  {stat_str}', end=" ")
+                    phy_str = ""
+                ps = size_str(int(prg*current_resource.total_size))
+                ts = size_str(current_resource.total_size)
+                ss = size_str(speed, "b")
+                stat_str = f"{percent}% - {ps} of {ts} - {ss}ps{phy_str}"
+                if prg != 1.0:
+                    print(f"{erase_str}Transferring file {syms[i]} {stat_str}", end=es)
+                else:
+                    print(f"{erase_str}Transfer complete  {stat_str}", end=es)
             else:
-                print(f'\r                                                                                  \rWaiting for transfer to start {syms[i]} ', end=" ")
+                print(f"{erase_str}Waiting for transfer to start {syms[i]} ", end=es)
             sys.stdout.flush()
             i = (i+1)%len(syms)
 
@@ -488,13 +530,12 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
         if silent:
             print("The transfer failed")
         else:
-            print("\r                                                            \rThe transfer failed")
+            print(f"{erase_str}The transfer failed")
         exit(1)
     else:
         if silent:
             print(f"{file} fetched from {RNS.prettyhexrep(destination_hash)}")
         else:
-            #print("\r                                                                                  \r"+str(file)+" fetched from "+RNS.prettyhexrep(destination_hash))
             print(f"\n{file} fetched from {RNS.prettyhexrep(destination_hash)}")
         link.teardown()
         time.sleep(0.15)
@@ -504,10 +545,11 @@ def fetch(configdir, verbosity = 0, quietness = 0, destination = None, file = No
     exit(0)
 
 
-def send(configdir, verbosity = 0, quietness = 0, destination = None, file = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT, silent=False):
+def send(configdir, verbosity = 0, quietness = 0, destination = None, file = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT, silent=False, phy_rates=False):
-    global current_resource, resource_done, link, speed
+    global current_resource, resource_done, link, speed, show_phy_rates
     from tempfile import TemporaryFile
     targetloglevel = 3+verbosity-quietness
+    show_phy_rates = phy_rates
 
     try:
         dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
@@ -536,14 +578,14 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         print("Filename exceeds max size, cannot send")
         exit(1)
     else:
-        print("Preparing file...", end=" ")
+        print("Preparing file...", end=es)
 
     temp_file.write(filename_len.to_bytes(2, "big"))
     temp_file.write(filename_bytes)
     temp_file.write(real_file.read())
     temp_file.seek(0)
 
-    print("\r                                                            \r", end="")
+    print(f"{erase_str}", end="")
 
     reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel)
 
@@ -566,7 +608,7 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         if silent:
             print(f"Path to {RNS.prettyhexrep(destination_hash)} requested")
         else:
-            print(f"Path to {RNS.prettyhexrep(destination_hash)} requested  ", end=" ")
+            print(f"Path to {RNS.prettyhexrep(destination_hash)} requested  ", end=es)
         sys.stdout.flush()
 
     i = 0
@@ -583,13 +625,13 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         if silent:
             print("Path not found")
         else:
-            print("\r                                                            \rPath not found")
+            print(f"{erase_str}Path not found")
         exit(1)
     else:
         if silent:
             print(f"Establishing link with {RNS.prettyhexrep(destination_hash)}")
         else:
-            print(f'\r                                                            \rEstablishing link with {RNS.prettyhexrep(destination_hash)} ', end=" ")
+            print(f"{erase_str}Establishing link with {RNS.prettyhexrep(destination_hash)} ", end=es)
 
     receiver_identity = RNS.Identity.recall(destination_hash)
     receiver_destination = RNS.Destination(
@@ -612,19 +654,19 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         if silent:
             print(f"Link establishment with {RNS.prettyhexrep(destination_hash)} timed out")
         else:
-            print(f'\r                                                            \rLink establishment with {RNS.prettyhexrep(destination_hash)} timed out')
+            print(f"{erase_str}Link establishment with {RNS.prettyhexrep(destination_hash)} timed out")
         exit(1)
     elif not RNS.Transport.has_path(destination_hash):
         if silent:
             print(f"No path found to {RNS.prettyhexrep(destination_hash)}")
         else:
-            print(f'\r                                                            \rNo path found to {RNS.prettyhexrep(destination_hash)}')
+            print(f"{erase_str}No path found to {RNS.prettyhexrep(destination_hash)}")
         exit(1)
     else:
         if silent:
             print("Advertising file resource...")
         else:
-            print("\r                                                            \rAdvertising file resource  ", end=" ")
+            print(f"{erase_str}Advertising file resource  ", end=es)
 
     link.identify(identity)
     resource = RNS.Resource(temp_file, link, callback = sender_progress, progress_callback = sender_progress)
@@ -642,23 +684,32 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         if silent:
             print(f"File was not accepted by {RNS.prettyhexrep(destination_hash)}")
         else:
-            print(f'\r                                                            \rFile was not accepted by {RNS.prettyhexrep(destination_hash)}')
+            print(f"{erase_str}File was not accepted by {RNS.prettyhexrep(destination_hash)}")
         exit(1)
     else:
         if silent:
             print("Transferring file...")
         else:
-            print("\r                                                            \rTransferring file  ", end=" ")
+            print(f"{erase_str}Transferring file  ", end=es)
 
     def progress_update(i, done=False):
         time.sleep(0.1)
         prg = current_resource.get_progress()
         percent = round(prg * 100.0, 1)
-        stat_str = f"{percent}% - {size_str(int(prg * current_resource.total_size))} of {size_str(current_resource.total_size)} - {size_str(speed, 'b')}ps"
+        if show_phy_rates:
-        if not done:
+            pss = size_str(phy_speed, "b")
-            print(f'\r                                                                                  \rTransferring file {syms[i]} {stat_str}', end=" ")
+            phy_str = f" ({pss}ps at physical layer)"
         else:
-            print(f'\r                                                                                  \rTransfer complete  {stat_str}', end=" ")
+            phy_str = ""
+        es = "  "
+        cs = size_str(int(prg*current_resource.total_size))
+        ts = size_str(current_resource.total_size)
+        ss = size_str(speed, "b")
+        stat_str = f"{percent}% - {cs} of {ts} - {ss}ps{phy_str}"
+        if not done:
+            print(f"{erase_str}Transferring file {syms[i]} {stat_str}", end=es)
+        else:
+            print(f"{erase_str}Transfer complete  {stat_str}", end=es)
         sys.stdout.flush()
         i = (i+1)%len(syms)
         return i
@@ -674,13 +725,12 @@ def send(configdir, verbosity = 0, quietness = 0, destination = None, file = Non
         if silent:
             print("The transfer failed")
         else:
-            print("\r                                                            \rThe transfer failed")
+            print(f"{erase_str}The transfer failed")
         exit(1)
     else:
         if silent:
             print(f"{file_path} copied to {RNS.prettyhexrep(destination_hash)}")
         else:
-            # print("\r                                                                                  \r"+str(file_path)+" copied to "+RNS.prettyhexrep(destination_hash))
             print(f"\n{file_path} copied to {RNS.prettyhexrep(destination_hash)}")
         link.teardown()
         time.sleep(0.25)
@@ -706,6 +756,7 @@ def main():
         parser.add_argument('-n', '--no-auth', action='store_true', default=False, help="accept requests from anyone")
         parser.add_argument('-p', '--print-identity', action='store_true', default=False, help="print identity and destination info and exit")
         parser.add_argument("-w", action="store", metavar="seconds", type=float, help="sender timeout before giving up", default=RNS.Transport.PATH_REQUEST_TIMEOUT)
+        parser.add_argument('-P', '--phy-rates', action='store_true', default=False, help="display physical layer transfer rates")
         # parser.add_argument("--limit", action="store", metavar="files", type=float, help="maximum number of files to accept", default=None)
         parser.add_argument("--version", action="version", version=f"rncp {__version__}")
 
@@ -735,6 +786,7 @@ def main():
                     file = args.file,
                     timeout = args.w,
                     silent = args.silent,
+                    phy_rates = args.phy_rates,
                 )
             else:
                 print("")
@@ -750,6 +802,7 @@ def main():
                 file = args.file,
                 timeout = args.w,
                 silent = args.silent,
+                phy_rates = args.phy_rates,
             )
 
         else:

RNS/_version.py

@@ -1 +1 @@
-__version__ = "0.8.2"
+__version__ = "0.8.3"