Skip to content

Commit 17c8efd

Browse files
committed
gh-159717: Use atomic groups for stars in glob.translate
glob.translate() dropped the star indices from fnmatch._translate(), so every star in a segment became a plain backtracking group and matching took exponential time in pathlib's match, full_match and glob. Each interior star and the fixed part after it are now wrapped in an atomic group, as fnmatch already does. Pairs whose fixed part can match a separator are left unchanged, so the set of matching names stays the same.
1 parent 457ba33 commit 17c8efd

4 files changed

Lines changed: 64 additions & 4 deletions

File tree

‎Lib/glob.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -324,13 +324,35 @@ def translate(pat, *, recursive=False, include_hidden=False, seps=None):
324324
if part:
325325
if not include_hidden and part[0] in '*?':
326326
results.append(r'(?!\.)')
327-
results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)[0])
327+
results.extend(_translate_segment(part, not_sep, seps))
328328
if idx < last_part_idx:
329329
results.append(any_sep)
330330
res = ''.join(results)
331331
return fr'(?s:{res})\z'
332332

333333

334+
def _translate_segment(part, not_sep, seps):
335+
# Wrap each interior `STAR fixed` pair in an atomic group to prevent
336+
# catastrophic backtracking, as fnmatch._join_translated_parts() does.
337+
# A pair whose fixed part can match a separator (a character class) is
338+
# left as it is, since the atomic group could change what it matches.
339+
star = f'{not_sep}*'
340+
parts, star_indices = fnmatch._translate(part, star, not_sep)
341+
if not star_indices:
342+
return parts
343+
res = parts[:star_indices[0]]
344+
for i, j in zip(star_indices, star_indices[1:]):
345+
fixed = parts[i + 1:j]
346+
if any(re.match(p, sep) for p in fixed for sep in seps):
347+
res.extend(parts[i:j])
348+
else:
349+
res.append(f'(?>{star}?')
350+
res.extend(fixed)
351+
res.append(')')
352+
res.extend(parts[star_indices[-1]:])
353+
return res
354+
355+
334356
@functools.lru_cache(maxsize=512)
335357
def _compile_pattern(pat, seps, case_sensitive, recursive=True):
336358
"""Compile given glob pattern to a re.Pattern object (observing case

‎Lib/test/test_glob.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,21 @@ def test_translate_matching(self):
425425
self.assertIsNotNone(match(os.path.join('foo', 'bar.txt')))
426426
self.assertIsNone(match(os.path.join('foo', '.bar.txt')))
427427

428+
def test_translate_slow(self):
429+
match = re.compile(glob.translate('*a' * 8 + '*b', seps='/')).match
430+
self.assertIsNotNone(match('a' * 100 + 'b'))
431+
# The next "takes forever" if the regexp translation is
432+
# straightforward. See bpo-40480.
433+
self.assertIsNone(match('a' * 100))
434+
435+
def test_translate_negated_class_matches_sep(self):
436+
# A character class can match a separator, so each star is not
437+
# limited to a single segment there.
438+
match = re.compile(glob.translate('*[!x]*x', seps='/')).match
439+
self.assertIsNotNone(match('a/x'))
440+
self.assertIsNotNone(match('ab/x'))
441+
self.assertIsNone(match('a/b/x'))
442+
428443
def test_translate(self):
429444
def fn(pat):
430445
return glob.translate(pat, seps='/')
@@ -443,7 +458,7 @@ def fn(pat):
443458
self.assertEqual(fn('a**'), r'(?s:a[^/]*)\z')
444459
self.assertEqual(fn('**b'), r'(?s:(?!\.)[^/]*b)\z')
445460
self.assertEqual(fn('/**/*/*.*/**'),
446-
r'(?s:/(?!\.)[^/]*/[^/.][^/]*/(?!\.)[^/]*\.[^/]*/(?!\.)[^/]*)\z')
461+
r'(?s:/(?!\.)[^/]*/[^/.][^/]*/(?!\.)(?>[^/]*?\.)[^/]*/(?!\.)[^/]*)\z')
447462

448463
def test_translate_include_hidden(self):
449464
def fn(pat):
@@ -462,7 +477,7 @@ def fn(pat):
462477
self.assertEqual(fn('***'), r'(?s:[^/]*)\z')
463478
self.assertEqual(fn('a**'), r'(?s:a[^/]*)\z')
464479
self.assertEqual(fn('**b'), r'(?s:[^/]*b)\z')
465-
self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/[^/]*/[^/]+/[^/]*\.[^/]*/[^/]*)\z')
480+
self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/[^/]*/[^/]+/(?>[^/]*?\.)[^/]*/[^/]*)\z')
466481

467482
def test_translate_recursive(self):
468483
def fn(pat):
@@ -474,7 +489,7 @@ def fn(pat):
474489
self.assertEqual(fn('***'), r'(?s:[^/]*)\z')
475490
self.assertEqual(fn('a**'), r'(?s:a[^/]*)\z')
476491
self.assertEqual(fn('**b'), r'(?s:[^/]*b)\z')
477-
self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/(?:.+/)?[^/]+/[^/]*\.[^/]*/.*)\z')
492+
self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/(?:.+/)?[^/]+/(?>[^/]*?\.)[^/]*/.*)\z')
478493

479494
def test_translate_seps(self):
480495
def fn(pat):

‎Lib/test/test_pathlib/test_pathlib.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -549,6 +549,16 @@ def test_full_match_case_sensitive(self):
549549
self.assertFalse(P('c:/a/B.Py').full_match('C:/A/*.pY', case_sensitive=True))
550550
self.assertTrue(P('/a/b/c.py').full_match('/A/*/*.Py', case_sensitive=False))
551551

552+
def test_full_match_slow(self):
553+
P = self.cls
554+
pat = '*a' * 8 + '*b'
555+
self.assertTrue(P('a' * 100 + 'b').full_match(pat))
556+
self.assertTrue(P('a' * 100 + 'b').match(pat))
557+
# The next "takes forever" if the regexp translation is
558+
# straightforward. See bpo-40480.
559+
self.assertFalse(P('a' * 100).full_match(pat))
560+
self.assertFalse(P('a' * 100).match(pat))
561+
552562
def test_match_empty(self):
553563
P = self.cls
554564
self.assertRaises(ValueError, P('a').match, '')
@@ -2973,6 +2983,14 @@ def test_glob_windows(self):
29732983
self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") })
29742984
self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") })
29752985

2986+
def test_glob_slow(self):
2987+
P = self.cls
2988+
p = P(self.base)
2989+
q = p / ('a' * 100)
2990+
q.touch()
2991+
self.assertEqual(list(p.glob('*a' * 8 + '*b')), [])
2992+
self.assertEqual(list(p.glob('*a' * 8)), [q])
2993+
29762994
def test_glob_empty_pattern(self):
29772995
p = self.cls('')
29782996
with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'):
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Fix exponential matching time in :func:`glob.translate`,
2+
:meth:`pathlib.PurePath.match`, :meth:`~pathlib.PurePath.full_match` and
3+
:meth:`pathlib.Path.glob` for patterns with several ``*`` wildcards in one
4+
segment. Interior ``*`` wildcards are now translated to atomic groups where
5+
this cannot change which names match, as :func:`fnmatch.translate` does.

0 commit comments

Comments
 (0)