Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/_pytest/pathlib.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import atexit
2import contextlib
3import fnmatch
4import importlib.util
5import itertools
6import os
7import shutil
8import sys
9import uuid
10import warnings
11from enum import Enum
12from functools import partial
13from os.path import expanduser
14from os.path import expandvars
15from os.path import isabs
16from os.path import sep
17from posixpath import sep as posix_sep
18from types import ModuleType
19from typing import Iterable
20from typing import Iterator
21from typing import Optional
22from typing import Set
23from typing import TypeVar
24from typing import Union
26import py
28from _pytest.compat import assert_never
29from _pytest.outcomes import skip
30from _pytest.warning_types import PytestWarning
32if sys.version_info[:2] >= (3, 6):
33 from pathlib import Path, PurePath
34else:
35 from pathlib2 import Path, PurePath
37__all__ = ["Path", "PurePath"]
40LOCK_TIMEOUT = 60 * 60 * 3
43_AnyPurePath = TypeVar("_AnyPurePath", bound=PurePath)
46def get_lock_path(path: _AnyPurePath) -> _AnyPurePath:
47 return path.joinpath(".lock")
50def ensure_reset_dir(path: Path) -> None:
51 """
52 ensures the given path is an empty directory
53 """
54 if path.exists():
55 rm_rf(path)
56 path.mkdir()
59def on_rm_rf_error(func, path: str, exc, *, start_path: Path) -> bool:
60 """Handles known read-only errors during rmtree.
62 The returned value is used only by our own tests.
63 """
64 exctype, excvalue = exc[:2]
66 # another process removed the file in the middle of the "rm_rf" (xdist for example)
67 # more context: https://github.com/pytest-dev/pytest/issues/5974#issuecomment-543799018
68 if isinstance(excvalue, FileNotFoundError):
69 return False
71 if not isinstance(excvalue, PermissionError):
72 warnings.warn(
73 PytestWarning(
74 "(rm_rf) error removing {}\n{}: {}".format(path, exctype, excvalue)
75 )
76 )
77 return False
79 if func not in (os.rmdir, os.remove, os.unlink):
80 if func not in (os.open,):
81 warnings.warn(
82 PytestWarning(
83 "(rm_rf) unknown function {} when removing {}:\n{}: {}".format(
84 func, path, exctype, excvalue
85 )
86 )
87 )
88 return False
90 # Chmod + retry.
91 import stat
93 def chmod_rw(p: str) -> None:
94 mode = os.stat(p).st_mode
95 os.chmod(p, mode | stat.S_IRUSR | stat.S_IWUSR)
97 # For files, we need to recursively go upwards in the directories to
98 # ensure they all are also writable.
99 p = Path(path)
100 if p.is_file():
101 for parent in p.parents:
102 chmod_rw(str(parent))
103 # stop when we reach the original path passed to rm_rf
104 if parent == start_path:
105 break
106 chmod_rw(str(path))
108 func(path)
109 return True
112def ensure_extended_length_path(path: Path) -> Path:
113 """Get the extended-length version of a path (Windows).
115 On Windows, by default, the maximum length of a path (MAX_PATH) is 260
116 characters, and operations on paths longer than that fail. But it is possible
117 to overcome this by converting the path to "extended-length" form before
118 performing the operation:
119 https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation
121 On Windows, this function returns the extended-length absolute version of path.
122 On other platforms it returns path unchanged.
123 """
124 if sys.platform.startswith("win32"):
125 path = path.resolve()
126 path = Path(get_extended_length_path_str(str(path)))
127 return path
130def get_extended_length_path_str(path: str) -> str:
131 """Converts to extended length path as a str"""
132 long_path_prefix = "\\\\?\\"
133 unc_long_path_prefix = "\\\\?\\UNC\\"
134 if path.startswith((long_path_prefix, unc_long_path_prefix)):
135 return path
136 # UNC
137 if path.startswith("\\\\"):
138 return unc_long_path_prefix + path[2:]
139 return long_path_prefix + path
142def rm_rf(path: Path) -> None:
143 """Remove the path contents recursively, even if some elements
144 are read-only.
145 """
146 path = ensure_extended_length_path(path)
147 onerror = partial(on_rm_rf_error, start_path=path)
148 shutil.rmtree(str(path), onerror=onerror)
151def find_prefixed(root: Path, prefix: str) -> Iterator[Path]:
152 """finds all elements in root that begin with the prefix, case insensitive"""
153 l_prefix = prefix.lower()
154 for x in root.iterdir():
155 if x.name.lower().startswith(l_prefix):
156 yield x
159def extract_suffixes(iter: Iterable[PurePath], prefix: str) -> Iterator[str]:
160 """
161 :param iter: iterator over path names
162 :param prefix: expected prefix of the path names
163 :returns: the parts of the paths following the prefix
164 """
165 p_len = len(prefix)
166 for p in iter:
167 yield p.name[p_len:]
170def find_suffixes(root: Path, prefix: str) -> Iterator[str]:
171 """combines find_prefixes and extract_suffixes
172 """
173 return extract_suffixes(find_prefixed(root, prefix), prefix)
176def parse_num(maybe_num) -> int:
177 """parses number path suffixes, returns -1 on error"""
178 try:
179 return int(maybe_num)
180 except ValueError:
181 return -1
184def _force_symlink(
185 root: Path, target: Union[str, PurePath], link_to: Union[str, Path]
186) -> None:
187 """helper to create the current symlink
189 it's full of race conditions that are reasonably ok to ignore
190 for the context of best effort linking to the latest test run
192 the presumption being that in case of much parallelism
193 the inaccuracy is going to be acceptable
194 """
195 current_symlink = root.joinpath(target)
196 try:
197 current_symlink.unlink()
198 except OSError:
199 pass
200 try:
201 current_symlink.symlink_to(link_to)
202 except Exception:
203 pass
206def make_numbered_dir(root: Path, prefix: str) -> Path:
207 """create a directory with an increased number as suffix for the given prefix"""
208 for i in range(10):
209 # try up to 10 times to create the folder
210 max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1)
211 new_number = max_existing + 1
212 new_path = root.joinpath("{}{}".format(prefix, new_number))
213 try:
214 new_path.mkdir()
215 except Exception:
216 pass
217 else:
218 _force_symlink(root, prefix + "current", new_path)
219 return new_path
220 else:
221 raise OSError(
222 "could not create numbered dir with prefix "
223 "{prefix} in {root} after 10 tries".format(prefix=prefix, root=root)
224 )
227def create_cleanup_lock(p: Path) -> Path:
228 """crates a lock to prevent premature folder cleanup"""
229 lock_path = get_lock_path(p)
230 try:
231 fd = os.open(str(lock_path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
232 except FileExistsError as e:
233 raise OSError("cannot create lockfile in {path}".format(path=p)) from e
234 else:
235 pid = os.getpid()
236 spid = str(pid).encode()
237 os.write(fd, spid)
238 os.close(fd)
239 if not lock_path.is_file():
240 raise OSError("lock path got renamed after successful creation")
241 return lock_path
244def register_cleanup_lock_removal(lock_path: Path, register=atexit.register):
245 """registers a cleanup function for removing a lock, by default on atexit"""
246 pid = os.getpid()
248 def cleanup_on_exit(lock_path: Path = lock_path, original_pid: int = pid) -> None:
249 current_pid = os.getpid()
250 if current_pid != original_pid:
251 # fork
252 return
253 try:
254 lock_path.unlink()
255 except OSError:
256 pass
258 return register(cleanup_on_exit)
261def maybe_delete_a_numbered_dir(path: Path) -> None:
262 """removes a numbered directory if its lock can be obtained and it does not seem to be in use"""
263 path = ensure_extended_length_path(path)
264 lock_path = None
265 try:
266 lock_path = create_cleanup_lock(path)
267 parent = path.parent
269 garbage = parent.joinpath("garbage-{}".format(uuid.uuid4()))
270 path.rename(garbage)
271 rm_rf(garbage)
272 except OSError:
273 # known races:
274 # * other process did a cleanup at the same time
275 # * deletable folder was found
276 # * process cwd (Windows)
277 return
278 finally:
279 # if we created the lock, ensure we remove it even if we failed
280 # to properly remove the numbered dir
281 if lock_path is not None:
282 try:
283 lock_path.unlink()
284 except OSError:
285 pass
288def ensure_deletable(path: Path, consider_lock_dead_if_created_before: float) -> bool:
289 """checks if `path` is deletable based on whether the lock file is expired"""
290 if path.is_symlink():
291 return False
292 lock = get_lock_path(path)
293 try:
294 if not lock.is_file():
295 return True
296 except OSError:
297 # we might not have access to the lock file at all, in this case assume
298 # we don't have access to the entire directory (#7491).
299 return False
300 try:
301 lock_time = lock.stat().st_mtime
302 except Exception:
303 return False
304 else:
305 if lock_time < consider_lock_dead_if_created_before:
306 # wa want to ignore any errors while trying to remove the lock such as:
307 # - PermissionDenied, like the file permissions have changed since the lock creation
308 # - FileNotFoundError, in case another pytest process got here first.
309 # and any other cause of failure.
310 with contextlib.suppress(OSError):
311 lock.unlink()
312 return True
313 return False
316def try_cleanup(path: Path, consider_lock_dead_if_created_before: float) -> None:
317 """tries to cleanup a folder if we can ensure it's deletable"""
318 if ensure_deletable(path, consider_lock_dead_if_created_before):
319 maybe_delete_a_numbered_dir(path)
322def cleanup_candidates(root: Path, prefix: str, keep: int) -> Iterator[Path]:
323 """lists candidates for numbered directories to be removed - follows py.path"""
324 max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1)
325 max_delete = max_existing - keep
326 paths = find_prefixed(root, prefix)
327 paths, paths2 = itertools.tee(paths)
328 numbers = map(parse_num, extract_suffixes(paths2, prefix))
329 for path, number in zip(paths, numbers):
330 if number <= max_delete:
331 yield path
334def cleanup_numbered_dir(
335 root: Path, prefix: str, keep: int, consider_lock_dead_if_created_before: float
336) -> None:
337 """cleanup for lock driven numbered directories"""
338 for path in cleanup_candidates(root, prefix, keep):
339 try_cleanup(path, consider_lock_dead_if_created_before)
340 for path in root.glob("garbage-*"):
341 try_cleanup(path, consider_lock_dead_if_created_before)
344def make_numbered_dir_with_cleanup(
345 root: Path, prefix: str, keep: int, lock_timeout: float
346) -> Path:
347 """creates a numbered dir with a cleanup lock and removes old ones"""
348 e = None
349 for i in range(10):
350 try:
351 p = make_numbered_dir(root, prefix)
352 lock_path = create_cleanup_lock(p)
353 register_cleanup_lock_removal(lock_path)
354 except Exception as exc:
355 e = exc
356 else:
357 consider_lock_dead_if_created_before = p.stat().st_mtime - lock_timeout
358 # Register a cleanup for program exit
359 atexit.register(
360 cleanup_numbered_dir,
361 root,
362 prefix,
363 keep,
364 consider_lock_dead_if_created_before,
365 )
366 return p
367 assert e is not None
368 raise e
371def resolve_from_str(input: str, root: py.path.local) -> Path:
372 assert not isinstance(input, Path), "would break on py2"
373 rootpath = Path(root)
374 input = expanduser(input)
375 input = expandvars(input)
376 if isabs(input):
377 return Path(input)
378 else:
379 return rootpath.joinpath(input)
382def fnmatch_ex(pattern: str, path) -> bool:
383 """FNMatcher port from py.path.common which works with PurePath() instances.
385 The difference between this algorithm and PurePath.match() is that the latter matches "**" glob expressions
386 for each part of the path, while this algorithm uses the whole path instead.
388 For example:
389 "tests/foo/bar/doc/test_foo.py" matches pattern "tests/**/doc/test*.py" with this algorithm, but not with
390 PurePath.match().
392 This algorithm was ported to keep backward-compatibility with existing settings which assume paths match according
393 this logic.
395 References:
396 * https://bugs.python.org/issue29249
397 * https://bugs.python.org/issue34731
398 """
399 path = PurePath(path)
400 iswin32 = sys.platform.startswith("win")
402 if iswin32 and sep not in pattern and posix_sep in pattern:
403 # Running on Windows, the pattern has no Windows path separators,
404 # and the pattern has one or more Posix path separators. Replace
405 # the Posix path separators with the Windows path separator.
406 pattern = pattern.replace(posix_sep, sep)
408 if sep not in pattern:
409 name = path.name
410 else:
411 name = str(path)
412 if path.is_absolute() and not os.path.isabs(pattern):
413 pattern = "*{}{}".format(os.sep, pattern)
414 return fnmatch.fnmatch(name, pattern)
417def parts(s: str) -> Set[str]:
418 parts = s.split(sep)
419 return {sep.join(parts[: i + 1]) or sep for i in range(len(parts))}
422def symlink_or_skip(src, dst, **kwargs):
423 """Makes a symlink or skips the test in case symlinks are not supported."""
424 try:
425 os.symlink(str(src), str(dst), **kwargs)
426 except OSError as e:
427 skip("symlinks not supported: {}".format(e))
430class ImportMode(Enum):
431 """Possible values for `mode` parameter of `import_path`"""
433 prepend = "prepend"
434 append = "append"
435 importlib = "importlib"
438class ImportPathMismatchError(ImportError):
439 """Raised on import_path() if there is a mismatch of __file__'s.
441 This can happen when `import_path` is called multiple times with different filenames that has
442 the same basename but reside in packages
443 (for example "/tests1/test_foo.py" and "/tests2/test_foo.py").
444 """
447def import_path(
448 p: Union[str, py.path.local, Path],
449 *,
450 mode: Union[str, ImportMode] = ImportMode.prepend
451) -> ModuleType:
452 """
453 Imports and returns a module from the given path, which can be a file (a module) or
454 a directory (a package).
456 The import mechanism used is controlled by the `mode` parameter:
458 * `mode == ImportMode.prepend`: the directory containing the module (or package, taking
459 `__init__.py` files into account) will be put at the *start* of `sys.path` before
460 being imported with `__import__.
462 * `mode == ImportMode.append`: same as `prepend`, but the directory will be appended
463 to the end of `sys.path`, if not already in `sys.path`.
465 * `mode == ImportMode.importlib`: uses more fine control mechanisms provided by `importlib`
466 to import the module, which avoids having to use `__import__` and muck with `sys.path`
467 at all. It effectively allows having same-named test modules in different places.
469 :raise ImportPathMismatchError: if after importing the given `path` and the module `__file__`
470 are different. Only raised in `prepend` and `append` modes.
471 """
472 mode = ImportMode(mode)
474 path = Path(str(p))
476 if not path.exists():
477 raise ImportError(path)
479 if mode is ImportMode.importlib:
480 module_name = path.stem
482 for meta_importer in sys.meta_path:
483 spec = meta_importer.find_spec(module_name, [str(path.parent)])
484 if spec is not None:
485 break
486 else:
487 spec = importlib.util.spec_from_file_location(module_name, str(path))
489 if spec is None:
490 raise ImportError(
491 "Can't find module {} at location {}".format(module_name, str(path))
492 )
493 mod = importlib.util.module_from_spec(spec)
494 spec.loader.exec_module(mod) # type: ignore[union-attr]
495 return mod
497 pkg_path = resolve_package_path(path)
498 if pkg_path is not None:
499 pkg_root = pkg_path.parent
500 names = list(path.with_suffix("").relative_to(pkg_root).parts)
501 if names[-1] == "__init__":
502 names.pop()
503 module_name = ".".join(names)
504 else:
505 pkg_root = path.parent
506 module_name = path.stem
508 # change sys.path permanently: restoring it at the end of this function would cause surprising
509 # problems because of delayed imports: for example, a conftest.py file imported by this function
510 # might have local imports, which would fail at runtime if we restored sys.path.
511 if mode is ImportMode.append:
512 if str(pkg_root) not in sys.path:
513 sys.path.append(str(pkg_root))
514 elif mode is ImportMode.prepend:
515 if str(pkg_root) != sys.path[0]:
516 sys.path.insert(0, str(pkg_root))
517 else:
518 assert_never(mode)
520 importlib.import_module(module_name)
522 mod = sys.modules[module_name]
523 if path.name == "__init__.py":
524 return mod
526 ignore = os.environ.get("PY_IGNORE_IMPORTMISMATCH", "")
527 if ignore != "1":
528 module_file = mod.__file__
529 if module_file.endswith((".pyc", ".pyo")):
530 module_file = module_file[:-1]
531 if module_file.endswith(os.path.sep + "__init__.py"):
532 module_file = module_file[: -(len(os.path.sep + "__init__.py"))]
534 try:
535 is_same = os.path.samefile(str(path), module_file)
536 except FileNotFoundError:
537 is_same = False
539 if not is_same:
540 raise ImportPathMismatchError(module_name, module_file, path)
542 return mod
545def resolve_package_path(path: Path) -> Optional[Path]:
546 """Return the Python package path by looking for the last
547 directory upwards which still contains an __init__.py.
548 Return None if it can not be determined.
549 """
550 result = None
551 for parent in itertools.chain((path,), path.parents):
552 if parent.is_dir():
553 if not parent.joinpath("__init__.py").is_file():
554 break
555 if not parent.name.isidentifier():
556 break
557 result = parent
558 return result