diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eea76e..a3977d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ All notable changes to this project will be documented in this file ## Unreleased +- Fixed `rspace_client.eln.fs.GalleryFilesystem` reading only the first page of a folder + listing: it failed to open (`StopIteration`) when the Gallery folder was not on the first + page of the Home folder, and `listdir` showed only the newest 20 entries of a Gallery folder. + Both broke Galaxy's shipped RSpace file source. - Gallery upload routing and section-mismatch handling (PR #56): clearer `GallerySectionMismatch` exception, optional `on_mismatch="reroute"` policy to auto-reroute uploads into the server-chosen section, `upload()` now diff --git a/rspace_client/eln/fs.py b/rspace_client/eln/fs.py index 060c098..4f09cf9 100644 --- a/rspace_client/eln/fs.py +++ b/rspace_client/eln/fs.py @@ -3,7 +3,7 @@ from fs.base import FS from rspace_client.eln import eln from rspace_client.client_base import ClientBase -from typing import Optional, List, Text, BinaryIO, Mapping, Any +from typing import Optional, List, Text, BinaryIO, Mapping, Any, Iterator from fs.info import Info from fs.permissions import Permissions from fs.subfs import SubFS @@ -145,7 +145,18 @@ def __init__(self, server: str, api_key: str, on_mismatch: str = ON_MISMATCH_RAI ) self.on_mismatch = on_mismatch self.eln_client = eln.ELNClient(server, api_key) - self.gallery_id = next(file['id'] for file in self.eln_client.list_folder_tree()['records'] if file['name'] == 'Gallery') + self.gallery_id = next(file['id'] for file in self._all_records(None) if file['name'] == 'Gallery') + + def _all_records(self, folder_id) -> Iterator[dict]: + """All records of a folder listing, following its 'next' links: the Home folder + of a busy account holds more than one page, and Gallery need not be on the first.""" + page = self.eln_client.list_folder_tree(folder_id) + while True: + yield from page.get('records', []) + # not link_exists(): that raises when a page carries no _links at all + if not any(link.get('rel') == 'next' for link in page.get('_links') or []): + return + page = self.eln_client.get_link_contents(page, 'next') def getinfo(self, path, namespaces=None) -> Info: is_file = path.split('/')[-1][:2] == "GL" @@ -170,7 +181,7 @@ def getinfo(self, path, namespaces=None) -> Info: def listdir(self, path: Text) -> List[Text]: id = path in [u'.', u'/', u'./'] and self.gallery_id or path_to_id(path) - return [file['globalId'] for file in self.eln_client.list_folder_tree(id)['records']] + return [file['globalId'] for file in self._all_records(id)] def makedir(self, path: Text, permissions: Optional[Permissions] = None, recreate: bool = False) -> SubFS[FS]: new_folder_name = path.split('/')[-1] diff --git a/rspace_client/tests/eln_fs_test.py b/rspace_client/tests/eln_fs_test.py index 1e290fc..4fd54a6 100644 --- a/rspace_client/tests/eln_fs_test.py +++ b/rspace_client/tests/eln_fs_test.py @@ -366,3 +366,42 @@ def test_invalid_per_call_policy_rejected(self): if __name__ == '__main__': unittest.main() + + +def _page(records, next_page=None): + links = [{'rel': 'next', 'link': next_page}] if next_page else [] + return {'records': records, '_links': links} + + +class ElnFilesystemPagingTest(unittest.TestCase): + """Galaxy's shipped RSpace file source failed for accounts whose Home folder held more + than one page: Gallery was looked up on the first page only.""" + + HOME = { + None: _page([{'id': i, 'name': f'tree-{i}', 'globalId': f'FL{i}'} for i in range(20)], 'p2'), + 'p2': _page([{'id': 123, 'name': 'Gallery', 'globalId': 'GF123'}]), + } + GALLERY = { + None: _page([{'id': 1, 'name': 'Images', 'globalId': 'GF1'}], 'g2'), + 'g2': _page([{'id': 2, 'name': 'Documents', 'globalId': 'GF2'}]), + } + + def setUp(self): + def list_folder_tree(folder_id=None, *args, **kwargs): + return (self.GALLERY if folder_id == 123 else self.HOME)[None] + + def get_link_contents(response, rel): + link = response['_links'][0]['link'] + return {**self.HOME, **self.GALLERY}[link] + + for name, fake in (('list_folder_tree', list_folder_tree), ('get_link_contents', get_link_contents)): + patcher = patch(f'rspace_client.eln.eln.ELNClient.{name}', side_effect=fake) + patcher.start() + self.addCleanup(patcher.stop) + self.fs = GalleryFilesystem("https://example.com", "api_key") + + def test_gallery_found_beyond_first_page_of_home(self): + self.assertEqual(123, self.fs.gallery_id) + + def test_listdir_follows_next_links(self): + self.assertEqual(['GF1', 'GF2'], self.fs.listdir('/'))