From 0b623ff98b25de2d4a37df8d7da2edbf70304c7d Mon Sep 17 00:00:00 2001 From: Tilo Mathes Date: Thu, 8 Oct 2026 17:39:16 +0200 Subject: [PATCH] GalleryFilesystem follows paginated folder listings GalleryFilesystem looked for the Gallery folder on the first page of the Home folder listing only, and listdir returned only the first page of a folder. With RSpace's default page size of 20 and newest-first ordering, the filesystem failed to open (StopIteration) once a user had about 14 items of their own in Home, and listings of Gallery folders with more than 20 entries were silently truncated. Galaxy's shipped RSpace file source is built on this class, so both showed up there. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +++ rspace_client/eln/fs.py | 17 ++++++++++--- rspace_client/tests/eln_fs_test.py | 39 ++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eea76e..a3977d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ All notable changes to this project will be documented in this file ## Unreleased +- Fixed `rspace_client.eln.fs.GalleryFilesystem` reading only the first page of a folder + listing: it failed to open (`StopIteration`) when the Gallery folder was not on the first + page of the Home folder, and `listdir` showed only the newest 20 entries of a Gallery folder. + Both broke Galaxy's shipped RSpace file source. - Gallery upload routing and section-mismatch handling (PR #56): clearer `GallerySectionMismatch` exception, optional `on_mismatch="reroute"` policy to auto-reroute uploads into the server-chosen section, `upload()` now diff --git a/rspace_client/eln/fs.py b/rspace_client/eln/fs.py index 060c098..4f09cf9 100644 --- a/rspace_client/eln/fs.py +++ b/rspace_client/eln/fs.py @@ -3,7 +3,7 @@ from fs.base import FS from rspace_client.eln import eln from rspace_client.client_base import ClientBase -from typing import Optional, List, Text, BinaryIO, Mapping, Any +from typing import Optional, List, Text, BinaryIO, Mapping, Any, Iterator from fs.info import Info from fs.permissions import Permissions from fs.subfs import SubFS @@ -145,7 +145,18 @@ def __init__(self, server: str, api_key: str, on_mismatch: str = ON_MISMATCH_RAI ) self.on_mismatch = on_mismatch self.eln_client = eln.ELNClient(server, api_key) - self.gallery_id = next(file['id'] for file in self.eln_client.list_folder_tree()['records'] if file['name'] == 'Gallery') + self.gallery_id = next(file['id'] for file in self._all_records(None) if file['name'] == 'Gallery') + + def _all_records(self, folder_id) -> Iterator[dict]: + """All records of a folder listing, following its 'next' links: the Home folder + of a busy account holds more than one page, and Gallery need not be on the first.""" + page = self.eln_client.list_folder_tree(folder_id) + while True: + yield from page.get('records', []) + # not link_exists(): that raises when a page carries no _links at all + if not any(link.get('rel') == 'next' for link in page.get('_links') or []): + return + page = self.eln_client.get_link_contents(page, 'next') def getinfo(self, path, namespaces=None) -> Info: is_file = path.split('/')[-1][:2] == "GL" @@ -170,7 +181,7 @@ def getinfo(self, path, namespaces=None) -> Info: def listdir(self, path: Text) -> List[Text]: id = path in [u'.', u'/', u'./'] and self.gallery_id or path_to_id(path) - return [file['globalId'] for file in self.eln_client.list_folder_tree(id)['records']] + return [file['globalId'] for file in self._all_records(id)] def makedir(self, path: Text, permissions: Optional[Permissions] = None, recreate: bool = False) -> SubFS[FS]: new_folder_name = path.split('/')[-1] diff --git a/rspace_client/tests/eln_fs_test.py b/rspace_client/tests/eln_fs_test.py index 1e290fc..4fd54a6 100644 --- a/rspace_client/tests/eln_fs_test.py +++ b/rspace_client/tests/eln_fs_test.py @@ -366,3 +366,42 @@ def test_invalid_per_call_policy_rejected(self): if __name__ == '__main__': unittest.main() + + +def _page(records, next_page=None): + links = [{'rel': 'next', 'link': next_page}] if next_page else [] + return {'records': records, '_links': links} + + +class ElnFilesystemPagingTest(unittest.TestCase): + """Galaxy's shipped RSpace file source failed for accounts whose Home folder held more + than one page: Gallery was looked up on the first page only.""" + + HOME = { + None: _page([{'id': i, 'name': f'tree-{i}', 'globalId': f'FL{i}'} for i in range(20)], 'p2'), + 'p2': _page([{'id': 123, 'name': 'Gallery', 'globalId': 'GF123'}]), + } + GALLERY = { + None: _page([{'id': 1, 'name': 'Images', 'globalId': 'GF1'}], 'g2'), + 'g2': _page([{'id': 2, 'name': 'Documents', 'globalId': 'GF2'}]), + } + + def setUp(self): + def list_folder_tree(folder_id=None, *args, **kwargs): + return (self.GALLERY if folder_id == 123 else self.HOME)[None] + + def get_link_contents(response, rel): + link = response['_links'][0]['link'] + return {**self.HOME, **self.GALLERY}[link] + + for name, fake in (('list_folder_tree', list_folder_tree), ('get_link_contents', get_link_contents)): + patcher = patch(f'rspace_client.eln.eln.ELNClient.{name}', side_effect=fake) + patcher.start() + self.addCleanup(patcher.stop) + self.fs = GalleryFilesystem("https://example.com", "api_key") + + def test_gallery_found_beyond_first_page_of_home(self): + self.assertEqual(123, self.fs.gallery_id) + + def test_listdir_follows_next_links(self): + self.assertEqual(['GF1', 'GF2'], self.fs.listdir('/'))