Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 28 additions & 10 deletions Lib/fnmatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ def translate(pat):
"""

parts, star_indices = _translate(pat, '*', '.')
return _join_translated_parts(parts, star_indices)
res = _join_translated_parts(parts, star_indices)
return fr'(?s:{res})\z'


_re_setops_sub = re.compile(r'([&~|])').sub
Expand Down Expand Up @@ -183,9 +184,15 @@ def _translate(pat, star, question_mark):
return res, star_indices


def _join_translated_parts(parts, star_indices):
def _join_translated_parts(parts, star_indices, star='.*', seps=()):
if not star_indices:
return fr'(?s:{"".join(parts)})\z'
return ''.join(parts)
if seps:
# glob has already split at each separator. Literal tokens only need
# checking for multi-character separators or case-insensitive uses.
check_literals = any(len(sep) != 1 or sep.lower() != sep.upper()
for sep in seps)
seps = ''.join(seps)
iter_star_indices = iter(star_indices)
j = next(iter_star_indices)
buffer = parts[:j] # fixed pieces at the start
Expand All @@ -194,16 +201,27 @@ def _join_translated_parts(parts, star_indices):
for j in iter_star_indices:
# Now deal with STAR fixed STAR fixed ...
# For an interior `STAR fixed` pairing, we want to do a minimal
# .*? match followed by `fixed`, with no possibility of backtracking.
# star match followed by `fixed`, with no possibility of backtracking.
# Atomic groups ("(?>...)") allow us to spell that directly.
# Note: people rely on the undocumented ability to join multiple
# translate() results together via "|" to build large regexps matching
# "one of many" shell patterns.
append('(?>.*?')
extend(parts[i:j])
append(')')
fixed = parts[i:j]
if seps and any(re.fullmatch(part, sep, re.DOTALL)
or re.fullmatch(part, sep, re.IGNORECASE | re.DOTALL)
for part in fixed
if check_literals or part == '.' or part.startswith('[')
for sep in seps):
# Unlike fnmatch stars, glob stars cannot consume separators.
# A character class in the fixed part may consume one, so its
# earliest match is not necessarily the one the suffix needs.
append(star)
extend(fixed)
else:
append(f'(?>{star}?')
extend(fixed)
append(')')
i = j + 1
append('.*')
append(star)
extend(parts[i:])
res = ''.join(buffer)
return fr'(?s:{res})\z'
return ''.join(buffer)
9 changes: 8 additions & 1 deletion Lib/glob.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,14 @@ def translate(pat, *, recursive=False, include_hidden=False, seps=None):
if part:
if not include_hidden and part[0] in '*?':
results.append(r'(?!\.)')
results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)[0])
translated, star_indices = fnmatch._translate(
part, f'{not_sep}*', not_sep)
if len(star_indices) < 2:
results.extend(translated)
else:
results.append(fnmatch._join_translated_parts(
translated, star_indices, star=f'{not_sep}*',
seps=seps))
if idx < last_part_idx:
results.append(any_sep)
res = ''.join(results)
Expand Down
51 changes: 48 additions & 3 deletions Lib/test/test_glob.py
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,51 @@ def test_translate_matching(self):
self.assertIsNotNone(match(os.path.join('foo', 'bar.txt')))
self.assertIsNone(match(os.path.join('foo', '.bar.txt')))

def test_translate_slow_pattern(self):
pattern = '*a' * 8 + '*b'
regex = glob.translate(pattern, seps='/')
self.assertIn('(?>', regex)
match = re.compile(regex).fullmatch
self.assertIsNotNone(match('a' * 8 + 'b'))
self.assertIsNone(match('a' * 8 + 'c'))

def test_translate_slow_pattern_with_separators(self):
def matches(pattern, path, *, seps='/', **kwargs):
regex = re.compile(glob.translate(pattern, seps=seps, **kwargs))
return regex.fullmatch(path) is not None

pattern = '*a*a/*b'
self.assertTrue(matches(pattern, 'baaa/ab'))
self.assertFalse(matches(pattern, 'ba/ab'))
self.assertFalse(matches(pattern, 'baaa/.ab'))
self.assertTrue(matches(pattern, 'baaa/.ab', include_hidden=True))

recursive_pattern = '**/*a*a/*b'
self.assertTrue(matches(recursive_pattern, 'dir/sub/baaa/ab', recursive=True))
self.assertFalse(matches(recursive_pattern, 'dir/.sub/baaa/ab', recursive=True))
self.assertTrue(matches(recursive_pattern, 'dir/.sub/baaa/ab',
recursive=True, include_hidden=True))

# A negated class can match an alphabetic separator with one case
# policy but not the other. Keep both uses of translate() working.
regex = glob.translate('*[!x]*x', seps='X')
self.assertIsNotNone(re.fullmatch(regex, '/X.bbb\\x'))
self.assertIsNotNone(re.fullmatch(regex, '/.bbb\\x', re.IGNORECASE))

self.assertTrue(matches('*a*a/*b', 'baaa\\ab', seps=['/', '\\']))
self.assertFalse(matches('*a*a/*b', 'ba/ab', seps=['/', '\\']))

# A character class can match a separator, so it cannot always be
# grouped atomically with the preceding wildcard.
self.assertTrue(matches('*[!x]*x', 'a/x'))

# These classes cannot match a separator and can be grouped safely.
self.assertTrue(matches('*[ab]*c', 'xac'))
self.assertFalse(matches('*[ab]*c', 'xa/bc'))
# A caret at the start of a character class is literal in glob syntax.
self.assertTrue(matches('*[^a]*b', '^b'))
self.assertFalse(matches('*[^a]*b', 'zb'))

def test_translate(self):
def fn(pat):
return glob.translate(pat, seps='/')
Expand All @@ -443,7 +488,7 @@ def fn(pat):
self.assertEqual(fn('a**'), r'(?s:a[^/]*)\z')
self.assertEqual(fn('**b'), r'(?s:(?!\.)[^/]*b)\z')
self.assertEqual(fn('/**/*/*.*/**'),
r'(?s:/(?!\.)[^/]*/[^/.][^/]*/(?!\.)[^/]*\.[^/]*/(?!\.)[^/]*)\z')
r'(?s:/(?!\.)[^/]*/[^/.][^/]*/(?!\.)(?>[^/]*?\.)[^/]*/(?!\.)[^/]*)\z')

def test_translate_include_hidden(self):
def fn(pat):
Expand All @@ -462,7 +507,7 @@ def fn(pat):
self.assertEqual(fn('***'), r'(?s:[^/]*)\z')
self.assertEqual(fn('a**'), r'(?s:a[^/]*)\z')
self.assertEqual(fn('**b'), r'(?s:[^/]*b)\z')
self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/[^/]*/[^/]+/[^/]*\.[^/]*/[^/]*)\z')
self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/[^/]*/[^/]+/(?>[^/]*?\.)[^/]*/[^/]*)\z')

def test_translate_recursive(self):
def fn(pat):
Expand All @@ -474,7 +519,7 @@ def fn(pat):
self.assertEqual(fn('***'), r'(?s:[^/]*)\z')
self.assertEqual(fn('a**'), r'(?s:a[^/]*)\z')
self.assertEqual(fn('**b'), r'(?s:[^/]*b)\z')
self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/(?:.+/)?[^/]+/[^/]*\.[^/]*/.*)\z')
self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/(?:.+/)?[^/]+/(?>[^/]*?\.)[^/]*/.*)\z')

def test_translate_seps(self):
def fn(pat):
Expand Down
21 changes: 21 additions & 0 deletions Lib/test/test_pathlib/test_pathlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -549,6 +549,16 @@ def test_full_match_case_sensitive(self):
self.assertFalse(P('c:/a/B.Py').full_match('C:/A/*.pY', case_sensitive=True))
self.assertTrue(P('/a/b/c.py').full_match('/A/*/*.Py', case_sensitive=False))

def test_slow_match(self):
P = self.cls
pattern = '*a' * 8 + '*b'
matching_path = P('a' * 8 + 'b')
non_matching_path = P('a' * 28)
self.assertTrue(matching_path.match(pattern))
self.assertTrue(matching_path.full_match(pattern))
self.assertFalse(non_matching_path.match(pattern))
self.assertFalse(non_matching_path.full_match(pattern))

def test_match_empty(self):
P = self.cls
self.assertRaises(ValueError, P('a').match, '')
Expand Down Expand Up @@ -2982,6 +2992,17 @@ def test_glob_empty_pattern(self):
with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'):
list(p.glob('./'))

def test_glob_slow_pattern(self):
P = self.cls
path = P(self.base)
pattern = '*a' * 8 + '*b'
matching_name = 'a' * 8 + 'b'
non_matching_name = 'a' * 28
matching_path = path / matching_name
matching_path.touch()
(path / non_matching_name).touch()
self.assertEqual(set(path.glob(pattern)), {matching_path})

def test_glob_many_open_files(self):
depth = 30
P = self.cls
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Avoid excessive backtracking in :func:`glob.translate` and the :mod:`pathlib`
matching methods for path segments containing multiple ``*`` wildcards.
Loading