GalleryFilesystem: follow paginated folder listings

CHANGELOG.md

@@ -8,6 +8,10 @@ All notable changes to this project will be documented in this file
 
 ## Unreleased
 
+- Fixed `rspace_client.eln.fs.GalleryFilesystem` reading only the first page of a folder
+  listing: it failed to open (`StopIteration`) when the Gallery folder was not on the first
+  page of the Home folder, and `listdir` showed only the newest 20 entries of a Gallery folder.
+  Both broke Galaxy's shipped RSpace file source.
 - Gallery upload routing and section-mismatch handling (PR #56): clearer
   `GallerySectionMismatch` exception, optional `on_mismatch="reroute"`
   policy to auto-reroute uploads into the server-chosen section, `upload()` now

rspace_client/eln/fs.py

@@ -3,7 +3,7 @@
 from fs.base import FS
 from rspace_client.eln import eln
 from rspace_client.client_base import ClientBase
-from typing import Optional, List, Text, BinaryIO, Mapping, Any
+from typing import Optional, List, Text, BinaryIO, Mapping, Any, Iterator
 from fs.info import Info
 from fs.permissions import Permissions
 from fs.subfs import SubFS
@@ -145,7 +145,18 @@ def __init__(self, server: str, api_key: str, on_mismatch: str = ON_MISMATCH_RAI
             )
         self.on_mismatch = on_mismatch
         self.eln_client = eln.ELNClient(server, api_key)
-        self.gallery_id = next(file['id'] for file in self.eln_client.list_folder_tree()['records'] if file['name'] == 'Gallery')
+        self.gallery_id = next(file['id'] for file in self._all_records(None) if file['name'] == 'Gallery')
+
+    def _all_records(self, folder_id) -> Iterator[dict]:
+        """All records of a folder listing, following its 'next' links: the Home folder
+        of a busy account holds more than one page, and Gallery need not be on the first."""
+        page = self.eln_client.list_folder_tree(folder_id)
+        while True:
+            yield from page.get('records', [])
+            # not link_exists(): that raises when a page carries no _links at all
+            if not any(link.get('rel') == 'next' for link in page.get('_links') or []):
+                return
+            page = self.eln_client.get_link_contents(page, 'next')
 
     def getinfo(self, path, namespaces=None) -> Info:
         is_file = path.split('/')[-1][:2] == "GL"
@@ -170,7 +181,7 @@ def getinfo(self, path, namespaces=None) -> Info:
 
     def listdir(self, path: Text) -> List[Text]:
         id = path in [u'.', u'/', u'./'] and self.gallery_id or path_to_id(path)
-        return [file['globalId'] for file in self.eln_client.list_folder_tree(id)['records']]
+        return [file['globalId'] for file in self._all_records(id)]
 
     def makedir(self, path: Text, permissions: Optional[Permissions] = None, recreate: bool = False) -> SubFS[FS]:
         new_folder_name = path.split('/')[-1]

rspace_client/tests/eln_fs_test.py

@@ -366,3 +366,42 @@ def test_invalid_per_call_policy_rejected(self):
 
 if __name__ == '__main__':
     unittest.main()
+
+
+def _page(records, next_page=None):
+    links = [{'rel': 'next', 'link': next_page}] if next_page else []
+    return {'records': records, '_links': links}
+
+
+class ElnFilesystemPagingTest(unittest.TestCase):
+    """Galaxy's shipped RSpace file source failed for accounts whose Home folder held more
+    than one page: Gallery was looked up on the first page only."""
+
+    HOME = {
+        None: _page([{'id': i, 'name': f'tree-{i}', 'globalId': f'FL{i}'} for i in range(20)], 'p2'),
+        'p2': _page([{'id': 123, 'name': 'Gallery', 'globalId': 'GF123'}]),
+    }
+    GALLERY = {
+        None: _page([{'id': 1, 'name': 'Images', 'globalId': 'GF1'}], 'g2'),
+        'g2': _page([{'id': 2, 'name': 'Documents', 'globalId': 'GF2'}]),
+    }
+
+    def setUp(self):
+        def list_folder_tree(folder_id=None, *args, **kwargs):
+            return (self.GALLERY if folder_id == 123 else self.HOME)[None]
+
+        def get_link_contents(response, rel):
+            link = response['_links'][0]['link']
+            return {**self.HOME, **self.GALLERY}[link]
+
+        for name, fake in (('list_folder_tree', list_folder_tree), ('get_link_contents', get_link_contents)):
+            patcher = patch(f'rspace_client.eln.eln.ELNClient.{name}', side_effect=fake)
+            patcher.start()
+            self.addCleanup(patcher.stop)
+        self.fs = GalleryFilesystem("https://example.com", "api_key")
+
+    def test_gallery_found_beyond_first_page_of_home(self):
+        self.assertEqual(123, self.fs.gallery_id)
+
+    def test_listdir_follows_next_links(self):
+        self.assertEqual(['GF1', 'GF2'], self.fs.listdir('/'))