From 8e742c7be18e325a50772c9d89d2934c30ebb3d5 Mon Sep 17 00:00:00 2001 From: harminius Date: Tue, 11 Aug 2026 14:22:06 +0200 Subject: [PATCH 1/9] add safeguard for too long path --- mergin/common.py | 3 +++ mergin/merginproject.py | 15 ++++++++++++++- mergin/utils.py | 17 ++++++++++++++++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/mergin/common.py b/mergin/common.py index 3d3f7e8..c3d11ca 100644 --- a/mergin/common.py +++ b/mergin/common.py @@ -33,6 +33,9 @@ # Maximum changes uploading to server MAX_UPLOAD_CHANGES = 100 +# maximum length of a path supported by Windows without long paths enabled (MAX_PATH) +WINDOWS_MAX_PATH = 260 + # default URL for submitting logs MERGIN_DEFAULT_LOGS_URL = "https://g4pfq226j0.execute-api.eu-west-1.amazonaws.com/mergin_client_log_submit" diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 12d798f..2237084 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -19,6 +19,7 @@ from .utils import ( generate_checksum, is_versioned_file, + is_path_too_long, int_version, do_sqlite_checkpoint, unique_path_name, @@ -623,8 +624,14 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]: delta_item.size = checkpoint_size delta_item.checksum = checkpoint_checksum + diff_location = self.fpath(diff_file, diff_directory) + if is_path_too_long(diff_location): + raise ClientError( + f"Cannot create changeset for '{path}': diff file path is too long " + f"({len(diff_location)} characters) for this OS: {diff_location}\n" + "Move the project to a directory with a shorter path and try again." + ) try: - diff_location = self.fpath(diff_file, diff_directory) self.geodiff.create_changeset(origin_file, current_file, diff_location) if not self.geodiff.has_changes(diff_location): os.remove(diff_location) @@ -677,6 +684,12 @@ def get_push_changes(self): diff_id = str(uuid.uuid4()) diff_name = path + "-diff-" + diff_id diff_file = self.fpath_meta(diff_name) + if is_path_too_long(diff_file): + raise ClientError( + f"Cannot create changeset for '{path}': diff file path is too long " + f"({len(diff_file)} characters) for this OS: {diff_file}\n" + "Move the project to a directory with a shorter path and try again." + ) try: self.geodiff.create_changeset(origin_file, current_file, diff_file) if self.geodiff.has_changes(diff_file): diff --git a/mergin/utils.py b/mergin/utils.py index 91796f3..6de05b3 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -9,7 +9,7 @@ import tempfile from enum import Enum from typing import Optional, Type, Union, ByteString -from .common import ClientError +from .common import ClientError, WINDOWS_MAX_PATH def generate_checksum(file, chunk_size=4096): @@ -266,6 +266,21 @@ def is_versioned_file(path: str) -> bool: return f_extension.lower() in diff_extensions +def is_path_too_long(path: str) -> bool: + """ + Check whether an absolute path is too long to be reliably created/opened on this OS. + + Windows limits paths to WINDOWS_MAX_PATH (260) characters unless long paths have been + explicitly enabled (which we cannot rely on being the case), so we treat that as the limit. + + :param path: absolute path to check + :type path: str + :returns: whether the path is likely to be rejected by the OS + :rtype: bool + """ + return os.name == "nt" and len(path) >= WINDOWS_MAX_PATH + + def is_qgis_file(path: str) -> bool: """ Check if file is a QGIS project file. From 77ae14ba1bbc455c66e57c8ecf9d05d09a49ad4e Mon Sep 17 00:00:00 2001 From: harminius Date: Tue, 11 Aug 2026 14:30:09 +0200 Subject: [PATCH 2/9] docstring --- mergin/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mergin/utils.py b/mergin/utils.py index 6de05b3..02bee94 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -270,9 +270,6 @@ def is_path_too_long(path: str) -> bool: """ Check whether an absolute path is too long to be reliably created/opened on this OS. - Windows limits paths to WINDOWS_MAX_PATH (260) characters unless long paths have been - explicitly enabled (which we cannot rely on being the case), so we treat that as the limit. - :param path: absolute path to check :type path: str :returns: whether the path is likely to be rejected by the OS From 6cc7fc6dd7fa7811a2fac639ec606fabaca90559 Mon Sep 17 00:00:00 2001 From: harminius Date: Fri, 14 Aug 2026 11:21:08 +0200 Subject: [PATCH 3/9] Add path length safeguards to pull and local create changeset --- mergin/client_pull.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 5210089..d9cce37 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -25,7 +25,7 @@ from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, save_to_file +from .utils import cleanup_tmp_dir, save_to_file, is_path_too_long from typing import List, Optional # status = download_project_async(...) @@ -480,6 +480,12 @@ def get_download_diff_files(delta_item: ProjectDeltaChange, target_dir: str) -> for diff in delta_item.diffs: dest_file_path = os.path.normpath(os.path.join(target_dir, diff.id)) + if is_path_too_long(dest_file_path): + raise ClientError( + f"Cannot download diff for '{delta_item.path}': diff file path is too long " + f"({len(dest_file_path)} characters) for this OS: {dest_file_path}\n" + "Move the project to a directory with a shorter path and try again." + ) download_items = get_download_items(delta_item.path, diff.size, diff.version, target_dir, diff.id, True) result.append(DownloadFile(dest_file_path, download_items)) return result @@ -574,12 +580,15 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: # if we have conflict and diff update, download the diff files if v2_pull_enabled: # using v2 endpoint to download diff files, without chunks. Then we are creating DownloadDiffQueueItem instances for each diff file. - diff_files.extend( - [ - DownloadDiffQueueItem(diff_item.id, os.path.join(tmp_dir.name, diff_item.id)) - for diff_item in change.diffs - ] - ) + for diff_item in change.diffs: + diff_path = os.path.join(tmp_dir.name, diff_item.id) + if is_path_too_long(diff_path): + raise ClientError( + f"Cannot download diff for '{change.path}': diff file path is too long " + f"({len(diff_path)} characters) for this OS: {diff_path}\n" + "Move the project to a directory with a shorter path and try again." + ) + diff_files.append(DownloadDiffQueueItem(diff_item.id, diff_path)) basefiles_to_patch.append((change.path, [diff.id for diff in change.diffs])) else: @@ -830,6 +839,12 @@ def download_diffs_async(mc, project_directory, file_path, versions): diff_only=True, ) dest_file_path = mp.fpath_cache(diff["path"], version=file["version"]) + if is_path_too_long(dest_file_path): + raise ClientError( + f"Cannot download diff for '{file.get('path')}': diff file path is too long " + f"({len(dest_file_path)} characters) for this OS: {dest_file_path}\n" + "Move the project to a directory with a shorter path and try again." + ) if os.path.exists(dest_file_path): continue download_files.append(DownloadFile(dest_file_path, items)) From 2d38d242820ec7b0aebc8d77c5f84af5b9cda22f Mon Sep 17 00:00:00 2001 From: harminius Date: Mon, 17 Aug 2026 22:53:32 +0200 Subject: [PATCH 4/9] Escape for windows long path --- mergin/client.py | 9 +- mergin/client_pull.py | 53 ++++------- mergin/client_push.py | 6 +- mergin/merginproject.py | 164 +++++++++++++++++--------------- mergin/report.py | 6 +- mergin/test/test_client_pull.py | 13 +-- mergin/utils.py | 29 +++++- 7 files changed, 150 insertions(+), 130 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index 1555051..8467a1c 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -67,6 +67,7 @@ int_version, is_version_acceptable, normalize_role, + long_path, ) from .version import __version__ @@ -1237,7 +1238,7 @@ def get_file_diff(self, project_dir, file_path, output_diff, version_from, versi # collect required versions from the cache diffs = [] for v in versions_to_fetch[1:]: - diffs.append(mp.fpath_cache(file_history["history"][v]["diff"]["path"], v)) + diffs.append(long_path(mp.fpath_cache(file_history["history"][v]["diff"]["path"], v))) # concatenate diffs, if needed output_dir = os.path.dirname(output_diff) @@ -1377,13 +1378,15 @@ def reset_local_changes(self, directory: str, files_to_reset: typing.List[str] = # remove all added files for file in push_changes["added"]: if all_files or file["path"] in files_to_reset: - os.remove(mp.fpath(file["path"])) + os.remove(long_path(mp.fpath(file["path"]))) # update files get override with previous version for file in push_changes["updated"]: if all_files or file["path"] in files_to_reset: if mp.is_versioned_file(file["path"]): - mp.geodiff.make_copy_sqlite(mp.fpath_meta(file["path"]), mp.fpath(file["path"])) + mp.geodiff.make_copy_sqlite( + long_path(mp.fpath_meta(file["path"])), long_path(mp.fpath(file["path"])) + ) else: files_download.append(file["path"]) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index d9cce37..2030457 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -25,7 +25,7 @@ from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, save_to_file, is_path_too_long +from .utils import cleanup_tmp_dir, save_to_file, long_path from typing import List, Optional # status = download_project_async(...) @@ -93,7 +93,9 @@ def __init__(self, file_path, size, version, diff_only, part_index, download_fil self.version = version # version of the file ("v123") self.diff_only = diff_only # whether downloading diff or full version self.part_index = part_index # index of the chunk - self.download_file_path = download_file_path # full path to a temporary file which will receive the content + self.download_file_path = long_path( + download_file_path + ) # full path to a temporary file which will receive the content def __repr__(self): return "".format( @@ -128,7 +130,9 @@ class DownloadDiffQueueItem: def __init__(self, diff_id, download_file_path): self.diff_id = diff_id # relative path to the file within project - self.download_file_path = download_file_path # full path to a temporary file which will receive the content + self.download_file_path = long_path( + download_file_path + ) # full path to a temporary file which will receive the content self.size = 0 # size of the item in bytes def __repr__(self): @@ -157,7 +161,7 @@ class DownloadFile: """ def __init__(self, dest_file, downloaded_items: typing.List[DownloadQueueItem], size_check=True): - self.dest_file = dest_file # full path to the destination file to be created + self.dest_file = long_path(dest_file) # full path to the destination file to be created self.downloaded_items = downloaded_items # list of pieces of the destination file to be merged self.size_check = size_check # whether we want to do merged file size check @@ -196,7 +200,7 @@ def get_download_items( items = [] for part_index in range(chunks): - download_file_path = os.path.join(file_dir, basename + ".{}".format(part_index)) + download_file_path = long_path(os.path.join(file_dir, basename + ".{}".format(part_index))) size = min(CHUNK_SIZE, file_size - part_index * CHUNK_SIZE) items.append(DownloadQueueItem(file_path, size, file_version, diff_only, part_index, download_file_path)) @@ -419,7 +423,7 @@ def apply(self, directory, mp): # Make a copy of the file to meta dir only if there is no user-specified path for the file. # destination_file is None for full project download and takes a meaningful value for a single file download. if mp.is_versioned_file(self.file_path) and self.destination_file is None: - mp.geodiff.make_copy_sqlite(mp.fpath(self.file_path), mp.fpath_meta(self.file_path)) + mp.geodiff.make_copy_sqlite(long_path(mp.fpath(self.file_path)), long_path(mp.fpath_meta(self.file_path))) class PullJob: @@ -479,13 +483,7 @@ def get_download_diff_files(delta_item: ProjectDeltaChange, target_dir: str) -> result = [] for diff in delta_item.diffs: - dest_file_path = os.path.normpath(os.path.join(target_dir, diff.id)) - if is_path_too_long(dest_file_path): - raise ClientError( - f"Cannot download diff for '{delta_item.path}': diff file path is too long " - f"({len(dest_file_path)} characters) for this OS: {dest_file_path}\n" - "Move the project to a directory with a shorter path and try again." - ) + dest_file_path = long_path(os.path.normpath(os.path.join(target_dir, diff.id))) download_items = get_download_items(delta_item.path, diff.size, diff.version, target_dir, diff.id, True) result.append(DownloadFile(dest_file_path, download_items)) return result @@ -561,7 +559,7 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: pull_action_type == PullActionType.COPY_CONFLICT and change.type == DeltaChangeType.UPDATE_DIFF ): basefile = mp.fpath_meta(change.path) - if not os.path.exists(basefile): + if not os.path.exists(long_path(basefile)): # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file # or we removed it within previous pull because we failed to apply patch the older version for some reason). # But it's not a problem - we will download the newest version and we're sorted. @@ -580,15 +578,12 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: # if we have conflict and diff update, download the diff files if v2_pull_enabled: # using v2 endpoint to download diff files, without chunks. Then we are creating DownloadDiffQueueItem instances for each diff file. - for diff_item in change.diffs: - diff_path = os.path.join(tmp_dir.name, diff_item.id) - if is_path_too_long(diff_path): - raise ClientError( - f"Cannot download diff for '{change.path}': diff file path is too long " - f"({len(diff_path)} characters) for this OS: {diff_path}\n" - "Move the project to a directory with a shorter path and try again." - ) - diff_files.append(DownloadDiffQueueItem(diff_item.id, diff_path)) + diff_files.extend( + [ + DownloadDiffQueueItem(diff_item.id, os.path.join(tmp_dir.name, diff_item.id)) + for diff_item in change.diffs + ] + ) basefiles_to_patch.append((change.path, [diff.id for diff in change.diffs])) else: @@ -731,7 +726,7 @@ def pull_project_finalize(job: PullJob): basefile = job.mp.fpath_meta(file_path) server_file = job.mp.fpath(file_path, job.tmp_dir.name) - shutil.copy(basefile, server_file) + shutil.copy(long_path(basefile), long_path(server_file)) diffs = [job.mp.fpath(f, job.tmp_dir.name) for f in file_diffs] patch_error = job.mp.apply_diffs(server_file, diffs) if patch_error: @@ -744,7 +739,7 @@ def pull_project_finalize(job: PullJob): job.mp.log.error("Diffs we were applying: " + str(diffs)) job.mp.log.error("Removing basefile because it would be corrupted anyway...") job.mp.log.info("--- pull aborted") - os.remove(basefile) + os.remove(long_path(basefile)) raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile)) conflicts = [] job.mp.log.info(f"--- applying pull actions {job.pull_actions}") @@ -838,13 +833,7 @@ def download_diffs_async(mc, project_directory, file_path, versions): download_path=diff.get("path"), diff_only=True, ) - dest_file_path = mp.fpath_cache(diff["path"], version=file["version"]) - if is_path_too_long(dest_file_path): - raise ClientError( - f"Cannot download diff for '{file.get('path')}': diff file path is too long " - f"({len(dest_file_path)} characters) for this OS: {dest_file_path}\n" - "Move the project to a directory with a shorter path and try again." - ) + dest_file_path = long_path(mp.fpath_cache(diff["path"], version=file["version"])) if os.path.exists(dest_file_path): continue download_files.append(DownloadFile(dest_file_path, items)) diff --git a/mergin/client_push.py b/mergin/client_push.py index 831b59b..bdcf0d1 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -34,7 +34,7 @@ ) from .merginproject import MerginProject, pygeodiff from .editor import filter_changes -from .utils import get_data_checksum, cleanup_tmp_dir +from .utils import get_data_checksum, cleanup_tmp_dir, long_path POST_JSON_HEADERS = {"Content-Type": "application/json"} @@ -114,7 +114,7 @@ def upload_chunk_v2_api(self, data: ByteString, checksum: str): self.mc.upload_chunks_cache.add(checksum, self.server_chunk_id) def upload_blocking(self): - with open(self.file_path, "rb") as file_handle: + with open(long_path(self.file_path), "rb") as file_handle: file_handle.seek(self.chunk_index * UPLOAD_CHUNK_SIZE) data = file_handle.read(UPLOAD_CHUNK_SIZE) checksum_str = get_data_checksum(data) @@ -507,7 +507,7 @@ def remove_diff_files(job: UploadJob) -> None: for change in job.changes.updated: diff = change.get_diff() if diff: - diff_file = job.mp.fpath_meta(diff.path) + diff_file = long_path(job.mp.fpath_meta(diff.path)) if os.path.exists(diff_file): os.remove(diff_file) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 2237084..d49870b 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -19,7 +19,7 @@ from .utils import ( generate_checksum, is_versioned_file, - is_path_too_long, + long_path, int_version, do_sqlite_checkpoint, unique_path_name, @@ -280,7 +280,7 @@ def is_gpkg_open(self, path): f_extension = os.path.splitext(path)[1] if f_extension != ".gpkg": return False - if os.path.exists(f"{path}-wal"): + if os.path.exists(f"{long_path(path)}-wal"): return True return False @@ -625,21 +625,16 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]: delta_item.checksum = checkpoint_checksum diff_location = self.fpath(diff_file, diff_directory) - if is_path_too_long(diff_location): - raise ClientError( - f"Cannot create changeset for '{path}': diff file path is too long " - f"({len(diff_location)} characters) for this OS: {diff_location}\n" - "Move the project to a directory with a shorter path and try again." - ) + diff_location_lp = long_path(diff_location) try: - self.geodiff.create_changeset(origin_file, current_file, diff_location) - if not self.geodiff.has_changes(diff_location): - os.remove(diff_location) + self.geodiff.create_changeset(long_path(origin_file), long_path(current_file), diff_location_lp) + if not self.geodiff.has_changes(diff_location_lp): + os.remove(diff_location_lp) continue delta_item.checksum = change.get("origin_checksum") delta_item.type = DeltaChangeType.UPDATE_DIFF - os.remove(diff_location) + os.remove(diff_location_lp) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to create changeset for " + path) # probably the database schema has been modified if geodiff cannot create changeset. @@ -684,28 +679,23 @@ def get_push_changes(self): diff_id = str(uuid.uuid4()) diff_name = path + "-diff-" + diff_id diff_file = self.fpath_meta(diff_name) - if is_path_too_long(diff_file): - raise ClientError( - f"Cannot create changeset for '{path}': diff file path is too long " - f"({len(diff_file)} characters) for this OS: {diff_file}\n" - "Move the project to a directory with a shorter path and try again." - ) + diff_file_lp = long_path(diff_file) try: - self.geodiff.create_changeset(origin_file, current_file, diff_file) - if self.geodiff.has_changes(diff_file): - diff_size = os.path.getsize(diff_file) + self.geodiff.create_changeset(long_path(origin_file), long_path(current_file), diff_file_lp) + if self.geodiff.has_changes(diff_file_lp): + diff_size = os.path.getsize(diff_file_lp) file["checksum"] = file["origin_checksum"] # need to match basefile on server file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(diff_size / UPLOAD_CHUNK_SIZE))] - file["mtime"] = datetime.fromtimestamp(os.path.getmtime(current_file), tzlocal()) + file["mtime"] = datetime.fromtimestamp(os.path.getmtime(long_path(current_file)), tzlocal()) file["diff"] = { "path": diff_name, - "checksum": generate_checksum(diff_file), + "checksum": generate_checksum(diff_file_lp), "size": diff_size, - "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file), tzlocal()), + "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file_lp), tzlocal()), } else: - if os.path.exists(diff_file): - os.remove(diff_file) + if os.path.exists(diff_file_lp): + os.remove(diff_file_lp) not_updated.append(file) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to create changeset for " + path) @@ -725,9 +715,10 @@ def copy_versioned_file_for_upload(self, f: FileChange, tmp_dir: str) -> str: self.log.info("Making a temporary copy (full upload): " + path) tmp_file = os.path.join(tmp_dir, path) os.makedirs(os.path.dirname(tmp_file), exist_ok=True) - self.geodiff.make_copy_sqlite(self.fpath(path), tmp_file) - f.size = os.path.getsize(tmp_file) - f.checksum = generate_checksum(tmp_file) + tmp_file_lp = long_path(tmp_file) + self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), tmp_file_lp) + f.size = os.path.getsize(tmp_file_lp) + f.checksum = generate_checksum(tmp_file_lp) f.chunks = [str(uuid.uuid4()) for i in range(math.ceil(f.size / UPLOAD_CHUNK_SIZE))] f.upload_file = tmp_file return tmp_file @@ -740,7 +731,7 @@ def get_list_of_push_changes(self, push_changes): changeset = self.fpath_meta(changeset_path) result_file = self.fpath("change_list" + str(idx), self.meta_dir) try: - self.geodiff.list_changes_summary(changeset, result_file) + self.geodiff.list_changes_summary(long_path(changeset), result_file) with open(result_file, "r") as f: change = f.read() changes[file["path"]] = json.loads(change) @@ -774,14 +765,17 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve server_file = self.fpath(path, download_dir) live_file = self.fpath(path) basefile = self.fpath_meta(path) + server_file_lp = long_path(server_file) + live_file_lp = long_path(live_file) + basefile_lp = long_path(basefile) action_type = action.type if action_type == PullActionType.COPY: # simply copy the file from server if is_versioned_file(path): - self.geodiff.make_copy_sqlite(server_file, live_file) - self.geodiff.make_copy_sqlite(server_file, basefile) + self.geodiff.make_copy_sqlite(server_file_lp, live_file_lp) + self.geodiff.make_copy_sqlite(server_file_lp, basefile_lp) else: - shutil.copy(server_file, live_file) + shutil.copy(server_file_lp, live_file_lp) elif action_type == PullActionType.APPLY_DIFF_NO_REBASE: # simply apply the diff without rebase (no local changes or non-conflicting local changes) self.update_without_rebase(path, server_file, live_file, basefile, download_dir) @@ -799,22 +793,22 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve conflicts.append(conflict) if self.is_versioned_file(path): try: - self.geodiff.make_copy_sqlite(server_file, live_file) - self.geodiff.make_copy_sqlite(server_file, basefile) + self.geodiff.make_copy_sqlite(server_file_lp, live_file_lp) + self.geodiff.make_copy_sqlite(server_file_lp, basefile_lp) except pygeodiff.GeoDiffLibError: self.log.info("failed to create SQLite copy for file: " + path) # create unfinished pull copy instead - f_server_unfinished = self.fpath_unfinished_pull(path) - self.geodiff.make_copy_sqlite(server_file, f_server_unfinished) + f_server_unfinished = long_path(self.fpath_unfinished_pull(path)) + self.geodiff.make_copy_sqlite(server_file_lp, f_server_unfinished) else: - shutil.copy(server_file, live_file) + shutil.copy(server_file_lp, live_file_lp) elif action_type == PullActionType.DELETE: # remove local file - if os.path.exists(live_file): - os.remove(live_file) - if self.is_versioned_file(path) and os.path.exists(basefile): - os.remove(basefile) + if os.path.exists(live_file_lp): + os.remove(live_file_lp) + if self.is_versioned_file(path) and os.path.exists(basefile_lp): + os.remove(basefile_lp) return conflicts @@ -841,51 +835,55 @@ def update_with_rebase(self, path, src, dest, basefile, temp_dir, user_name): """ self.log.info("updating file with rebase: " + path) - server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile - local_diff = self.fpath(f"{path}-local_diff", temp_dir) + src_lp = long_path(src) + dest_lp = long_path(dest) + basefile_lp = long_path(basefile) + + server_diff = long_path(self.fpath(f"{path}-server_diff", temp_dir)) # diff between server file and local basefile + local_diff = long_path(self.fpath(f"{path}-local_diff", temp_dir)) # temporary backup of file pulled from server for recovery - f_server_backup = self.fpath(f"{path}-server_backup", temp_dir) - self.geodiff.make_copy_sqlite(src, f_server_backup) + f_server_backup = long_path(self.fpath(f"{path}-server_backup", temp_dir)) + self.geodiff.make_copy_sqlite(src_lp, f_server_backup) # create temp backup (ideally with geodiff) of locally modified file if needed later - f_conflict_file = self.fpath(f"{path}-local_backup", temp_dir) + f_conflict_file = long_path(self.fpath(f"{path}-local_backup", temp_dir)) try: - self.geodiff.create_changeset(basefile, dest, local_diff) - self.geodiff.make_copy_sqlite(basefile, f_conflict_file) + self.geodiff.create_changeset(basefile_lp, dest_lp, local_diff) + self.geodiff.make_copy_sqlite(basefile_lp, f_conflict_file) self.geodiff.apply_changeset(f_conflict_file, local_diff) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): self.log.info("backup of local file with geodiff failed - need to do hard copy") - self.geodiff.make_copy_sqlite(dest, f_conflict_file) + self.geodiff.make_copy_sqlite(dest_lp, f_conflict_file) # in case there will be any conflicting operations found during rebase, # they will be stored in a JSON file - if there are no conflicts, the file # won't even be created - rebase_conflicts = unique_path_name( - edit_conflict_file_name(self.fpath(path), user_name, int_version(self.version())) + rebase_conflicts = long_path( + unique_path_name(edit_conflict_file_name(self.fpath(path), user_name, int_version(self.version()))) ) # try to do rebase magic try: - self.geodiff.create_changeset(basefile, src, server_diff) - self.geodiff.rebase(basefile, src, dest, rebase_conflicts) + self.geodiff.create_changeset(basefile_lp, src_lp, server_diff) + self.geodiff.rebase(basefile_lp, src_lp, dest_lp, rebase_conflicts) # make sure basefile is in the same state as remote server file (for calc of push changes) - self.geodiff.apply_changeset(basefile, server_diff) + self.geodiff.apply_changeset(basefile_lp, server_diff) self.log.info("rebase successful!") except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as err: self.log.warning("rebase failed! going to create conflict file") try: # it would not be possible to commit local changes, they need to end up in new conflict file - self.geodiff.make_copy_sqlite(f_conflict_file, dest) + self.geodiff.make_copy_sqlite(f_conflict_file, dest_lp) conflict = self.create_conflicted_copy(path, user_name) # original file synced with server - self.geodiff.make_copy_sqlite(f_server_backup, basefile) - self.geodiff.make_copy_sqlite(f_server_backup, dest) + self.geodiff.make_copy_sqlite(f_server_backup, basefile_lp) + self.geodiff.make_copy_sqlite(f_server_backup, dest_lp) return conflict except pygeodiff.GeoDiffLibError as err: self.log.warning("creation of conflicted copy failed! going to create an unfinished pull") - f_server_unfinished = self.fpath_unfinished_pull(path) + f_server_unfinished = long_path(self.fpath_unfinished_pull(path)) self.geodiff.make_copy_sqlite(f_server_backup, f_server_unfinished) return "" @@ -911,22 +909,27 @@ def update_without_rebase(self, path, src, dest, basefile, temp_dir): :type temp_dir: str """ self.log.info("updating file without rebase: " + path) + src_lp = long_path(src) + dest_lp = long_path(dest) + basefile_lp = long_path(basefile) try: - server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile + server_diff = long_path( + self.fpath(f"{path}-server_diff", temp_dir) + ) # diff between server file and local basefile # TODO: it could happen that basefile does not exist. # It was either never created (e.g. when pushing without geodiff) # or it was deleted by mistake(?) by the user. We should detect that # when starting pull and download it as well - self.geodiff.create_changeset(basefile, src, server_diff) - self.geodiff.apply_changeset(dest, server_diff) - self.geodiff.apply_changeset(basefile, server_diff) + self.geodiff.create_changeset(basefile_lp, src_lp, server_diff) + self.geodiff.apply_changeset(dest_lp, server_diff) + self.geodiff.apply_changeset(basefile_lp, server_diff) self.log.info("update successful") except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): self.log.warning("update failed! going to copy file") # something bad happened and we have failed to patch our local files - this should not happen if there # wasn't a schema change or something similar that geodiff can't handle. - self.geodiff.make_copy_sqlite(src, dest) - self.geodiff.make_copy_sqlite(src, basefile) + self.geodiff.make_copy_sqlite(src_lp, dest_lp) + self.geodiff.make_copy_sqlite(src_lp, basefile_lp) def apply_push_changes(self, changes): """ @@ -942,16 +945,17 @@ def apply_push_changes(self, changes): continue basefile = self.fpath_meta(path) + basefile_lp = long_path(basefile) if k == "removed": - os.remove(basefile) + os.remove(basefile_lp) elif k == "added": - self.geodiff.make_copy_sqlite(self.fpath(path), basefile) + self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), basefile_lp) elif k == "updated": # in case for geopackage cannot be created diff (e.g. forced update with committed changes from wal file) diff = item.get("diff") if not diff: self.log.info("updating basefile (copy) for: " + path) - self.geodiff.make_copy_sqlite(self.fpath(path), basefile) + self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), basefile_lp) else: self.log.info("updating basefile (diff) for: " + path) # better to apply diff to previous basefile to avoid issues with geodiff tmp files @@ -960,7 +964,7 @@ def apply_push_changes(self, changes): if patch_error: # in case of local sync issues it is safier to remove basefile, next time it will be downloaded from server self.log.warning("removing basefile (because of apply diff error) for: " + path) - os.remove(basefile) + os.remove(basefile_lp) else: pass @@ -974,7 +978,8 @@ def create_conflicted_copy(self, file: str, user_name: str): :rtype: str """ src = self.fpath(file) - if not os.path.exists(src): + src_lp = long_path(src) + if not os.path.exists(src_lp): return backup_path = unique_path_name( @@ -982,9 +987,9 @@ def create_conflicted_copy(self, file: str, user_name: str): ) if self.is_versioned_file(file): - self.geodiff.make_copy_sqlite(src, backup_path) + self.geodiff.make_copy_sqlite(src_lp, long_path(backup_path)) else: - shutil.copy(src, backup_path) + shutil.copy(src_lp, long_path(backup_path)) return backup_path def apply_diffs(self, basefile, diffs): @@ -1003,9 +1008,10 @@ def apply_diffs(self, basefile, diffs): if not self.is_versioned_file(basefile): return error + basefile_lp = long_path(basefile) for index, diff in enumerate(diffs): try: - self.geodiff.apply_changeset(basefile, diff) + self.geodiff.apply_changeset(basefile_lp, long_path(diff)) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to apply changeset " + diff + " to " + basefile) error = str(e) @@ -1055,12 +1061,12 @@ def resolve_unfinished_pull(self, user_name): self.log.info("resolving unfinished pull") - for root, dirs, files in os.walk(self.unfinished_pull_dir): + for root, dirs, files in os.walk(long_path(self.unfinished_pull_dir)): for file_name in files: - src = os.path.join(root, file_name) - file_path = os.path.relpath(src, self.unfinished_pull_dir) - dest = self.fpath(file_path) - basefile = self.fpath_meta(file_path) + src = os.path.join(root, file_name) # already long-path-prefixed, root came from os.walk above + file_path = os.path.relpath(src, long_path(self.unfinished_pull_dir)) + dest = long_path(self.fpath(file_path)) + basefile = long_path(self.fpath_meta(file_path)) self.log.info("trying to resolve unfinished pull for: " + file_path) @@ -1123,7 +1129,7 @@ def get_geodiff_changes_count(self, diff_rel_path: str): Never raises – diagnostics/logging must not fail. """ - diff_abs = self.fpath_meta(diff_rel_path) + diff_abs = long_path(self.fpath_meta(diff_rel_path)) try: return pygeodiff.GeoDiff().changes_count(diff_abs) except ( diff --git a/mergin/report.py b/mergin/report.py index 5b9cae4..abbf451 100644 --- a/mergin/report.py +++ b/mergin/report.py @@ -7,7 +7,7 @@ from . import ClientError from .merginproject import MerginProject, pygeodiff -from .utils import int_version +from .utils import int_version, long_path try: from qgis.core import ( @@ -243,7 +243,7 @@ def create_report(mc, directory, since, to, out_file): mc.download_file_diffs(directory, f["path"], history_keys) # download full gpkg in "to" version to analyze its schema to determine which col is geometry - full_gpkg = mp.fpath_cache(f["path"], version=to) + full_gpkg = long_path(mp.fpath_cache(f["path"], version=to)) if not os.path.exists(full_gpkg): mc.download_file(directory, f["path"], full_gpkg, to) @@ -263,7 +263,7 @@ def create_report(mc, directory, since, to, out_file): warnings.append(f"Missing diff: {f['path']} was {f['history'][version]['change']} in {version}") continue - v_diff_file = mp.fpath_cache(f["history"][version]["diff"]["path"], version=version) + v_diff_file = long_path(mp.fpath_cache(f["history"][version]["diff"]["path"], version=version)) version_data = versions_map[version] cr = mp.geodiff.read_changeset(v_diff_file) report = changeset_report(cr, schema, mp) diff --git a/mergin/test/test_client_pull.py b/mergin/test/test_client_pull.py index c20fe44..fda8bd0 100644 --- a/mergin/test/test_client_pull.py +++ b/mergin/test/test_client_pull.py @@ -4,6 +4,7 @@ from mergin.common import DeltaChangeType, CHUNK_SIZE from mergin.models import ProjectDeltaChange, ProjectDeltaItemDiff from mergin.client_pull import get_download_diff_files, get_download_items +from mergin.utils import long_path def test_get_diff_download_files(): @@ -25,7 +26,7 @@ def test_get_diff_download_files(): # Check diff f2 = files[0] - assert f2.dest_file == os.path.join(tmp_dir, "diff2") + assert f2.dest_file == long_path(os.path.join(tmp_dir, "diff2")) assert len(f2.downloaded_items) == 1 assert f2.downloaded_items[0].file_path == "data.gpkg" assert f2.downloaded_items[0].size == 20 @@ -41,7 +42,7 @@ def test_get_download_items(): assert items[0].file_path == "small.txt" assert items[0].size == 100 assert items[0].part_index == 0 - assert items[0].download_file_path == os.path.join(tmp_dir, "small.txt.0") + assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "small.txt.0")) # Case 2: Large file (multiple chunks) file_size = int(CHUNK_SIZE * 2.5) @@ -51,17 +52,17 @@ def test_get_download_items(): # Chunk 0 assert items[0].size == CHUNK_SIZE assert items[0].part_index == 0 - assert items[0].download_file_path == os.path.join(tmp_dir, "large.txt.0") + assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.0")) # Chunk 1 assert items[1].size == CHUNK_SIZE assert items[1].part_index == 1 - assert items[1].download_file_path == os.path.join(tmp_dir, "large.txt.1") + assert items[1].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.1")) # Chunk 2 assert items[2].size == int(CHUNK_SIZE * 0.5) assert items[2].part_index == 2 - assert items[2].download_file_path == os.path.join(tmp_dir, "large.txt.2") + assert items[2].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.2")) # Case 3: Empty file items = get_download_items("empty.txt", 0, "v1", tmp_dir) @@ -73,4 +74,4 @@ def test_get_download_items(): assert items[0].diff_only is True assert items[0].file_path == "base.gpkg" assert items[0].size == 50 - assert items[0].download_file_path == os.path.join(tmp_dir, "diff_file.0") + assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "diff_file.0")) diff --git a/mergin/utils.py b/mergin/utils.py index 02bee94..d1cdafc 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -88,10 +88,11 @@ def do_sqlite_checkpoint(path, log=None): """ new_size = None new_checksum = None - if ".gpkg" in path and os.path.exists(f"{path}-wal"): + path_lp = long_path(path) + if ".gpkg" in path and os.path.exists(f"{path_lp}-wal"): if log: log.info("checkpoint - going to add it in " + path) - conn = sqlite3.connect(path) + conn = sqlite3.connect(path_lp) cursor = conn.cursor() cursor.execute("PRAGMA wal_checkpoint=FULL") if log: @@ -99,8 +100,8 @@ def do_sqlite_checkpoint(path, log=None): cursor.execute("VACUUM") conn.commit() conn.close() - new_size = os.path.getsize(path) - new_checksum = generate_checksum(path) + new_size = os.path.getsize(path_lp) + new_checksum = generate_checksum(path_lp) if log: log.info("checkpoint - new size {} checksum {}".format(new_size, new_checksum)) @@ -278,6 +279,26 @@ def is_path_too_long(path: str) -> bool: return os.name == "nt" and len(path) >= WINDOWS_MAX_PATH +def long_path(path: str) -> str: + """ + Prefix an absolute path with the Windows "\\?\" extended-length marker, + so file APIs used by geodiff/SQLite and Python's own open() can handle paths longer + than MAX_PATH (260 characters) without raising an error. + + :param path: absolute or relative path, with either posix or windows separators + :type path: str + :returns: extended-length path on Windows, the unchanged path otherwise + :rtype: str + """ + if os.name != "nt": + return path + backslash = chr(92) + prefix = backslash + backslash + "?" + backslash + if path.startswith(prefix): + return path + return prefix + os.path.abspath(path) + + def is_qgis_file(path: str) -> bool: """ Check if file is a QGIS project file. From 096fa0a84a9aeac0b5bddb30b177bc1155bc5242 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 17 Aug 2026 23:04:58 +0200 Subject: [PATCH 5/9] black --- mergin/merginproject.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index d49870b..0757a96 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -839,7 +839,9 @@ def update_with_rebase(self, path, src, dest, basefile, temp_dir, user_name): dest_lp = long_path(dest) basefile_lp = long_path(basefile) - server_diff = long_path(self.fpath(f"{path}-server_diff", temp_dir)) # diff between server file and local basefile + server_diff = long_path( + self.fpath(f"{path}-server_diff", temp_dir) + ) # diff between server file and local basefile local_diff = long_path(self.fpath(f"{path}-local_diff", temp_dir)) # temporary backup of file pulled from server for recovery From d2bf5f255edbdb5360de9aa0f84fb446464caa74 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 17 Aug 2026 23:08:20 +0200 Subject: [PATCH 6/9] cleanup --- mergin/common.py | 3 --- mergin/utils.py | 19 +++---------------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/mergin/common.py b/mergin/common.py index c3d11ca..3d3f7e8 100644 --- a/mergin/common.py +++ b/mergin/common.py @@ -33,9 +33,6 @@ # Maximum changes uploading to server MAX_UPLOAD_CHANGES = 100 -# maximum length of a path supported by Windows without long paths enabled (MAX_PATH) -WINDOWS_MAX_PATH = 260 - # default URL for submitting logs MERGIN_DEFAULT_LOGS_URL = "https://g4pfq226j0.execute-api.eu-west-1.amazonaws.com/mergin_client_log_submit" diff --git a/mergin/utils.py b/mergin/utils.py index d1cdafc..251de54 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -9,7 +9,7 @@ import tempfile from enum import Enum from typing import Optional, Type, Union, ByteString -from .common import ClientError, WINDOWS_MAX_PATH +from .common import ClientError def generate_checksum(file, chunk_size=4096): @@ -267,23 +267,10 @@ def is_versioned_file(path: str) -> bool: return f_extension.lower() in diff_extensions -def is_path_too_long(path: str) -> bool: - """ - Check whether an absolute path is too long to be reliably created/opened on this OS. - - :param path: absolute path to check - :type path: str - :returns: whether the path is likely to be rejected by the OS - :rtype: bool - """ - return os.name == "nt" and len(path) >= WINDOWS_MAX_PATH - - def long_path(path: str) -> str: """ - Prefix an absolute path with the Windows "\\?\" extended-length marker, - so file APIs used by geodiff/SQLite and Python's own open() can handle paths longer - than MAX_PATH (260 characters) without raising an error. + Prefix an absolute path with the Windows "\\?\" extended-length marker, so file APIs used by + geodiff/SQLite and Python's own open() can handle long paths without raising an error. :param path: absolute or relative path, with either posix or windows separators :type path: str From e8c2f75d66378f57849e65862bd87c14fcf8d564 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 17 Aug 2026 23:17:42 +0200 Subject: [PATCH 7/9] black 2 --- mergin/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/utils.py b/mergin/utils.py index 251de54..b5bcac9 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -269,7 +269,7 @@ def is_versioned_file(path: str) -> bool: def long_path(path: str) -> str: """ - Prefix an absolute path with the Windows "\\?\" extended-length marker, so file APIs used by + Prefix an absolute path with the Windows "\\?\" extended-length marker, so file APIs used by geodiff/SQLite and Python's own open() can handle long paths without raising an error. :param path: absolute or relative path, with either posix or windows separators From 6bd80e30af4b8ee3e1a8781fcef2e9c01dc3c94b Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 18 Aug 2026 08:14:32 +0200 Subject: [PATCH 8/9] rm long path from tests --- mergin/test/test_client_pull.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/mergin/test/test_client_pull.py b/mergin/test/test_client_pull.py index fda8bd0..6bb8784 100644 --- a/mergin/test/test_client_pull.py +++ b/mergin/test/test_client_pull.py @@ -4,7 +4,6 @@ from mergin.common import DeltaChangeType, CHUNK_SIZE from mergin.models import ProjectDeltaChange, ProjectDeltaItemDiff from mergin.client_pull import get_download_diff_files, get_download_items -from mergin.utils import long_path def test_get_diff_download_files(): @@ -26,7 +25,7 @@ def test_get_diff_download_files(): # Check diff f2 = files[0] - assert f2.dest_file == long_path(os.path.join(tmp_dir, "diff2")) + assert f2.dest_file == os.path.join(tmp_dir, "diff2") assert len(f2.downloaded_items) == 1 assert f2.downloaded_items[0].file_path == "data.gpkg" assert f2.downloaded_items[0].size == 20 @@ -42,7 +41,7 @@ def test_get_download_items(): assert items[0].file_path == "small.txt" assert items[0].size == 100 assert items[0].part_index == 0 - assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "small.txt.0")) + assert items[0].download_file_path == os.path.join(tmp_dir, "small.txt.0") # Case 2: Large file (multiple chunks) file_size = int(CHUNK_SIZE * 2.5) @@ -52,17 +51,17 @@ def test_get_download_items(): # Chunk 0 assert items[0].size == CHUNK_SIZE assert items[0].part_index == 0 - assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.0")) + assert items[0].download_file_path == os.path.join(tmp_dir, "large.txt.0") # Chunk 1 assert items[1].size == CHUNK_SIZE assert items[1].part_index == 1 - assert items[1].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.1")) + assert items[1].download_file_path == (os.path.join(tmp_dir, "large.txt.1") # Chunk 2 assert items[2].size == int(CHUNK_SIZE * 0.5) assert items[2].part_index == 2 - assert items[2].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.2")) + assert items[2].download_file_path == os.path.join(tmp_dir, "large.txt.2") # Case 3: Empty file items = get_download_items("empty.txt", 0, "v1", tmp_dir) @@ -74,4 +73,4 @@ def test_get_download_items(): assert items[0].diff_only is True assert items[0].file_path == "base.gpkg" assert items[0].size == 50 - assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "diff_file.0")) + assert items[0].download_file_path == os.path.join(tmp_dir, "diff_file.0") From 782603c2b80c952d5d1ec113b419b21b037463fd Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 18 Aug 2026 08:15:05 +0200 Subject: [PATCH 9/9] rm long path from tests 2 --- mergin/test/test_client_pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/test/test_client_pull.py b/mergin/test/test_client_pull.py index 6bb8784..c20fe44 100644 --- a/mergin/test/test_client_pull.py +++ b/mergin/test/test_client_pull.py @@ -56,7 +56,7 @@ def test_get_download_items(): # Chunk 1 assert items[1].size == CHUNK_SIZE assert items[1].part_index == 1 - assert items[1].download_file_path == (os.path.join(tmp_dir, "large.txt.1") + assert items[1].download_file_path == os.path.join(tmp_dir, "large.txt.1") # Chunk 2 assert items[2].size == int(CHUNK_SIZE * 0.5)