Removed the Requirement to Install Python and NodeJS (Now Bundled with Borealis)

Dependencies/Python/Lib/test/test_peg_generator/__init__.py

@@ -0,0 +1,12 @@
+import os.path
+from test import support
+from test.support import load_package_tests
+
+
+# Creating a virtual environment and building C extensions is slow
+support.requires('cpu')
+
+
+# Load all tests in package
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)

Dependencies/Python/Lib/test/test_peg_generator/__main__.py

@@ -0,0 +1,4 @@
+import unittest
+from . import load_tests
+
+unittest.main()

Dependencies/Python/Lib/test/test_peg_generator/test_c_parser.py

@@ -0,0 +1,523 @@
+import contextlib
+import subprocess
+import sysconfig
+import textwrap
+import unittest
+import os
+import shutil
+import tempfile
+from pathlib import Path
+
+from test import test_tools
+from test import support
+from test.support import os_helper, import_helper
+from test.support.script_helper import assert_python_ok
+
+if support.check_cflags_pgo():
+    raise unittest.SkipTest("peg_generator test disabled under PGO build")
+
+test_tools.skip_if_missing("peg_generator")
+with test_tools.imports_under_tool("peg_generator"):
+    from pegen.grammar_parser import GeneratedParser as GrammarParser
+    from pegen.testutil import (
+        parse_string,
+        generate_parser_c_extension,
+        generate_c_parser_source,
+    )
+
+
+TEST_TEMPLATE = """
+tmp_dir = {extension_path!r}
+
+import ast
+import traceback
+import sys
+import unittest
+
+from test import test_tools
+with test_tools.imports_under_tool("peg_generator"):
+    from pegen.ast_dump import ast_dump
+
+sys.path.insert(0, tmp_dir)
+import parse
+
+class Tests(unittest.TestCase):
+
+    def check_input_strings_for_grammar(
+        self,
+        valid_cases = (),
+        invalid_cases = (),
+    ):
+        if valid_cases:
+            for case in valid_cases:
+                parse.parse_string(case, mode=0)
+
+        if invalid_cases:
+            for case in invalid_cases:
+                with self.assertRaises(SyntaxError):
+                    parse.parse_string(case, mode=0)
+
+    def verify_ast_generation(self, stmt):
+        expected_ast = ast.parse(stmt)
+        actual_ast = parse.parse_string(stmt, mode=1)
+        self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast))
+
+    def test_parse(self):
+        {test_source}
+
+unittest.main()
+"""
+
+
+@support.requires_subprocess()
+class TestCParser(unittest.TestCase):
+
+    _has_run = False
+
+    @classmethod
+    def setUpClass(cls):
+        if cls._has_run:
+            # Since gh-104798 (Use setuptools in peg-generator and reenable
+            # tests), this test case has been producing ref leaks. Initial
+            # debugging points to bug(s) in setuptools and/or importlib.
+            # See gh-105063 for more info.
+            raise unittest.SkipTest("gh-105063: can not rerun because of ref. leaks")
+        cls._has_run = True
+
+        # When running under regtest, a separate tempdir is used
+        # as the current directory and watched for left-overs.
+        # Reusing that as the base for temporary directories
+        # ensures everything is cleaned up properly and
+        # cleans up afterwards if not (with warnings).
+        cls.tmp_base = os.getcwd()
+        if os.path.samefile(cls.tmp_base, os_helper.SAVEDCWD):
+            cls.tmp_base = None
+        # Create a directory for the reuseable static library part of
+        # the pegen extension build process.  This greatly reduces the
+        # runtime overhead of spawning compiler processes.
+        cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base)
+        cls.addClassCleanup(shutil.rmtree, cls.library_dir)
+
+        with contextlib.ExitStack() as stack:
+            python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools_wheel("venv"))
+            sitepackages = subprocess.check_output(
+                [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"],
+                text=True,
+            ).strip()
+            stack.enter_context(import_helper.DirsOnSysPath(sitepackages))
+            cls.addClassCleanup(stack.pop_all().close)
+
+    @support.requires_venv_with_pip()
+    def setUp(self):
+        self._backup_config_vars = dict(sysconfig._CONFIG_VARS)
+        cmd = support.missing_compiler_executable()
+        if cmd is not None:
+            self.skipTest("The %r command is not found" % cmd)
+        self.old_cwd = os.getcwd()
+        self.tmp_path = tempfile.mkdtemp(dir=self.tmp_base)
+        self.enterContext(os_helper.change_cwd(self.tmp_path))
+
+    def tearDown(self):
+        os.chdir(self.old_cwd)
+        shutil.rmtree(self.tmp_path)
+        sysconfig._CONFIG_VARS.clear()
+        sysconfig._CONFIG_VARS.update(self._backup_config_vars)
+
+    def build_extension(self, grammar_source):
+        grammar = parse_string(grammar_source, GrammarParser)
+        # Because setUp() already changes the current directory to the
+        # temporary path, use a relative path here to prevent excessive
+        # path lengths when compiling.
+        generate_parser_c_extension(grammar, Path('.'), library_dir=self.library_dir)
+
+    def run_test(self, grammar_source, test_source):
+        self.build_extension(grammar_source)
+        test_source = textwrap.indent(textwrap.dedent(test_source), 8 * " ")
+        assert_python_ok(
+            "-c",
+            TEST_TEMPLATE.format(extension_path=self.tmp_path, test_source=test_source),
+        )
+
+    def test_c_parser(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a[asdl_stmt_seq*]=stmt* $ { _PyAST_Module(a, NULL, p->arena) }
+        stmt[stmt_ty]: a=expr_stmt { a }
+        expr_stmt[stmt_ty]: a=expression NEWLINE { _PyAST_Expr(a, EXTRA) }
+        expression[expr_ty]: ( l=expression '+' r=term { _PyAST_BinOp(l, Add, r, EXTRA) }
+                            | l=expression '-' r=term { _PyAST_BinOp(l, Sub, r, EXTRA) }
+                            | t=term { t }
+                            )
+        term[expr_ty]: ( l=term '*' r=factor { _PyAST_BinOp(l, Mult, r, EXTRA) }
+                    | l=term '/' r=factor { _PyAST_BinOp(l, Div, r, EXTRA) }
+                    | f=factor { f }
+                    )
+        factor[expr_ty]: ('(' e=expression ')' { e }
+                        | a=atom { a }
+                        )
+        atom[expr_ty]: ( n=NAME { n }
+                    | n=NUMBER { n }
+                    | s=STRING { s }
+                    )
+        """
+        test_source = """
+        expressions = [
+            "4+5",
+            "4-5",
+            "4*5",
+            "1+4*5",
+            "1+4/5",
+            "(1+1) + (1+1)",
+            "(1+1) - (1+1)",
+            "(1+1) * (1+1)",
+            "(1+1) / (1+1)",
+        ]
+
+        for expr in expressions:
+            the_ast = parse.parse_string(expr, mode=1)
+            expected_ast = ast.parse(expr)
+            self.assertEqual(ast_dump(the_ast), ast_dump(expected_ast))
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_lookahead(self) -> None:
+        grammar_source = """
+        start: NAME &NAME expr NEWLINE? ENDMARKER
+        expr: NAME | NUMBER
+        """
+        test_source = """
+        valid_cases = ["foo bar"]
+        invalid_cases = ["foo 34"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_negative_lookahead(self) -> None:
+        grammar_source = """
+        start: NAME !NAME expr NEWLINE? ENDMARKER
+        expr: NAME | NUMBER
+        """
+        test_source = """
+        valid_cases = ["foo 34"]
+        invalid_cases = ["foo bar"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_cut(self) -> None:
+        grammar_source = """
+        start: X ~ Y Z | X Q S
+        X: 'x'
+        Y: 'y'
+        Z: 'z'
+        Q: 'q'
+        S: 's'
+        """
+        test_source = """
+        valid_cases = ["x y z"]
+        invalid_cases = ["x q s"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_gather(self) -> None:
+        grammar_source = """
+        start: ';'.pass_stmt+ NEWLINE
+        pass_stmt: 'pass'
+        """
+        test_source = """
+        valid_cases = ["pass", "pass; pass"]
+        invalid_cases = ["pass;", "pass; pass;"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_left_recursion(self) -> None:
+        grammar_source = """
+        start: expr NEWLINE
+        expr: ('-' term | expr '+' term | term)
+        term: NUMBER
+        """
+        test_source = """
+        valid_cases = ["-34", "34", "34 + 12", "1 + 1 + 2 + 3"]
+        self.check_input_strings_for_grammar(valid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_advanced_left_recursive(self) -> None:
+        grammar_source = """
+        start: NUMBER | sign start
+        sign: ['-']
+        """
+        test_source = """
+        valid_cases = ["23", "-34"]
+        self.check_input_strings_for_grammar(valid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_mutually_left_recursive(self) -> None:
+        grammar_source = """
+        start: foo 'E'
+        foo: bar 'A' | 'B'
+        bar: foo 'C' | 'D'
+        """
+        test_source = """
+        valid_cases = ["B E", "D A C A E"]
+        self.check_input_strings_for_grammar(valid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_nasty_mutually_left_recursive(self) -> None:
+        grammar_source = """
+        start: target '='
+        target: maybe '+' | NAME
+        maybe: maybe '-' | target
+        """
+        test_source = """
+        valid_cases = ["x ="]
+        invalid_cases = ["x - + ="]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_return_stmt_noexpr_action(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        statements[asdl_stmt_seq*]: a[asdl_stmt_seq*]=statement+ { a }
+        statement[stmt_ty]: simple_stmt
+        simple_stmt[stmt_ty]: small_stmt
+        small_stmt[stmt_ty]: return_stmt
+        return_stmt[stmt_ty]: a='return' NEWLINE { _PyAST_Return(NULL, EXTRA) }
+        """
+        test_source = """
+        stmt = "return"
+        self.verify_ast_generation(stmt)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_gather_action_ast(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a[asdl_stmt_seq*]=';'.pass_stmt+ NEWLINE ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA)}
+        """
+        test_source = """
+        stmt = "pass; pass"
+        self.verify_ast_generation(stmt)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_pass_stmt_action(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        statements[asdl_stmt_seq*]: a[asdl_stmt_seq*]=statement+ { a }
+        statement[stmt_ty]: simple_stmt
+        simple_stmt[stmt_ty]: small_stmt
+        small_stmt[stmt_ty]: pass_stmt
+        pass_stmt[stmt_ty]: a='pass' NEWLINE { _PyAST_Pass(EXTRA) }
+        """
+        test_source = """
+        stmt = "pass"
+        self.verify_ast_generation(stmt)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_if_stmt_action(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) }
+        statement[asdl_stmt_seq*]:  a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } | simple_stmt
+
+        simple_stmt[asdl_stmt_seq*]: a=small_stmt b=further_small_stmt* [';'] NEWLINE {
+                                            (asdl_stmt_seq*)_PyPegen_seq_insert_in_front(p, a, b) }
+        further_small_stmt[stmt_ty]: ';' a=small_stmt { a }
+
+        block: simple_stmt | NEWLINE INDENT a=statements DEDENT { a }
+
+        compound_stmt: if_stmt
+
+        if_stmt: 'if' a=full_expression ':' b=block { _PyAST_If(a, b, NULL, EXTRA) }
+
+        small_stmt[stmt_ty]: pass_stmt
+
+        pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA) }
+
+        full_expression: NAME
+        """
+        test_source = """
+        stmt = "pass"
+        self.verify_ast_generation(stmt)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_same_name_different_types(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a[asdl_stmt_seq*]=import_from+ NEWLINE ENDMARKER { _PyAST_Module(a, NULL, p->arena)}
+        import_from[stmt_ty]: ( a='from' !'import' c=simple_name 'import' d=import_as_names_from {
+                                _PyAST_ImportFrom(c->v.Name.id, d, 0, EXTRA) }
+                            | a='from' '.' 'import' c=import_as_names_from {
+                                _PyAST_ImportFrom(NULL, c, 1, EXTRA) }
+                            )
+        simple_name[expr_ty]: NAME
+        import_as_names_from[asdl_alias_seq*]: a[asdl_alias_seq*]=','.import_as_name_from+ { a }
+        import_as_name_from[alias_ty]: a=NAME 'as' b=NAME { _PyAST_alias(((expr_ty) a)->v.Name.id, ((expr_ty) b)->v.Name.id, EXTRA) }
+        """
+        test_source = """
+        for stmt in ("from a import b as c", "from . import a as b"):
+            expected_ast = ast.parse(stmt)
+            actual_ast = parse.parse_string(stmt, mode=1)
+            self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast))
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_with_stmt_with_paren(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) }
+        statement[asdl_stmt_seq*]: a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) }
+        compound_stmt[stmt_ty]: with_stmt
+        with_stmt[stmt_ty]: (
+            a='with' '(' b[asdl_withitem_seq*]=','.with_item+ ')' ':' c=block {
+                _PyAST_With(b, (asdl_stmt_seq*) _PyPegen_singleton_seq(p, c), NULL, EXTRA) }
+        )
+        with_item[withitem_ty]: (
+            e=NAME o=['as' t=NAME { t }] { _PyAST_withitem(e, _PyPegen_set_expr_context(p, o, Store), p->arena) }
+        )
+        block[stmt_ty]: a=pass_stmt NEWLINE { a } | NEWLINE INDENT a=pass_stmt DEDENT { a }
+        pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA) }
+        """
+        test_source = """
+        stmt = "with (\\n    a as b,\\n    c as d\\n): pass"
+        the_ast = parse.parse_string(stmt, mode=1)
+        self.assertTrue(ast_dump(the_ast).startswith(
+            "Module(body=[With(items=[withitem(context_expr=Name(id='a', ctx=Load()), optional_vars=Name(id='b', ctx=Store())), "
+            "withitem(context_expr=Name(id='c', ctx=Load()), optional_vars=Name(id='d', ctx=Store()))]"
+        ))
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_ternary_operator(self) -> None:
+        grammar_source = """
+        start[mod_ty]: a=expr ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
+        expr[asdl_stmt_seq*]: a=listcomp NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, _PyAST_Expr(a, EXTRA)) }
+        listcomp[expr_ty]: (
+            a='[' b=NAME c=for_if_clauses d=']' { _PyAST_ListComp(b, c, EXTRA) }
+        )
+        for_if_clauses[asdl_comprehension_seq*]: (
+            a[asdl_comprehension_seq*]=(y=['async'] 'for' a=NAME 'in' b=NAME c[asdl_expr_seq*]=('if' z=NAME { z })*
+                { _PyAST_comprehension(_PyAST_Name(((expr_ty) a)->v.Name.id, Store, EXTRA), b, c, (y == NULL) ? 0 : 1, p->arena) })+ { a }
+        )
+        """
+        test_source = """
+        stmt = "[i for i in a if b]"
+        self.verify_ast_generation(stmt)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_syntax_error_for_string(self) -> None:
+        grammar_source = """
+        start: expr+ NEWLINE? ENDMARKER
+        expr: NAME
+        """
+        test_source = r"""
+        for text in ("a b 42 b a", "\u540d \u540d 42 \u540d \u540d"):
+            try:
+                parse.parse_string(text, mode=0)
+            except SyntaxError as e:
+                tb = traceback.format_exc()
+            self.assertTrue('File "<string>", line 1' in tb)
+            self.assertTrue(f"SyntaxError: invalid syntax" in tb)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_headers_and_trailer(self) -> None:
+        grammar_source = """
+        @header 'SOME HEADER'
+        @subheader 'SOME SUBHEADER'
+        @trailer 'SOME TRAILER'
+        start: expr+ NEWLINE? ENDMARKER
+        expr: x=NAME
+        """
+        grammar = parse_string(grammar_source, GrammarParser)
+        parser_source = generate_c_parser_source(grammar)
+
+        self.assertTrue("SOME HEADER" in parser_source)
+        self.assertTrue("SOME SUBHEADER" in parser_source)
+        self.assertTrue("SOME TRAILER" in parser_source)
+
+    def test_error_in_rules(self) -> None:
+        grammar_source = """
+        start: expr+ NEWLINE? ENDMARKER
+        expr: NAME {PyTuple_New(-1)}
+        """
+        # PyTuple_New raises SystemError if an invalid argument was passed.
+        test_source = """
+        with self.assertRaises(SystemError):
+            parse.parse_string("a", mode=0)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_no_soft_keywords(self) -> None:
+        grammar_source = """
+        start: expr+ NEWLINE? ENDMARKER
+        expr: 'foo'
+        """
+        grammar = parse_string(grammar_source, GrammarParser)
+        parser_source = generate_c_parser_source(grammar)
+        assert "expect_soft_keyword" not in parser_source
+
+    def test_soft_keywords(self) -> None:
+        grammar_source = """
+        start: expr+ NEWLINE? ENDMARKER
+        expr: "foo"
+        """
+        grammar = parse_string(grammar_source, GrammarParser)
+        parser_source = generate_c_parser_source(grammar)
+        assert "expect_soft_keyword" in parser_source
+
+    def test_soft_keywords_parse(self) -> None:
+        grammar_source = """
+        start: "if" expr '+' expr NEWLINE
+        expr: NAME
+        """
+        test_source = """
+        valid_cases = ["if if + if"]
+        invalid_cases = ["if if"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_soft_keywords_lookahead(self) -> None:
+        grammar_source = """
+        start: &"if" "if" expr '+' expr NEWLINE
+        expr: NAME
+        """
+        test_source = """
+        valid_cases = ["if if + if"]
+        invalid_cases = ["if if"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_forced(self) -> None:
+        grammar_source = """
+        start: NAME &&':' | NAME
+        """
+        test_source = """
+        self.assertEqual(parse.parse_string("number :", mode=0), None)
+        with self.assertRaises(SyntaxError) as e:
+            parse.parse_string("a", mode=0)
+        self.assertIn("expected ':'", str(e.exception))
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_forced_with_group(self) -> None:
+        grammar_source = """
+        start: NAME &&(':' | ';') | NAME
+        """
+        test_source = """
+        self.assertEqual(parse.parse_string("number :", mode=0), None)
+        self.assertEqual(parse.parse_string("number ;", mode=0), None)
+        with self.assertRaises(SyntaxError) as e:
+            parse.parse_string("a", mode=0)
+        self.assertIn("expected (':' | ';')", e.exception.args[0])
+        """
+        self.run_test(grammar_source, test_source)

Dependencies/Python/Lib/test/test_peg_generator/test_first_sets.py

@@ -0,0 +1,286 @@
+import unittest
+
+from test import test_tools
+from typing import Dict, Set
+
+test_tools.skip_if_missing("peg_generator")
+with test_tools.imports_under_tool("peg_generator"):
+    from pegen.grammar_parser import GeneratedParser as GrammarParser
+    from pegen.testutil import parse_string
+    from pegen.first_sets import FirstSetCalculator
+    from pegen.grammar import Grammar
+
+
+class TestFirstSets(unittest.TestCase):
+    def calculate_first_sets(self, grammar_source: str) -> Dict[str, Set[str]]:
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        return FirstSetCalculator(grammar.rules).calculate()
+
+    def test_alternatives(self) -> None:
+        grammar = """
+            start: expr NEWLINE? ENDMARKER
+            expr: A | B
+            A: 'a' | '-'
+            B: 'b' | '+'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "A": {"'a'", "'-'"},
+                "B": {"'+'", "'b'"},
+                "expr": {"'+'", "'a'", "'b'", "'-'"},
+                "start": {"'+'", "'a'", "'b'", "'-'"},
+            },
+        )
+
+    def test_optionals(self) -> None:
+        grammar = """
+            start: expr NEWLINE
+            expr: ['a'] ['b'] 'c'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "expr": {"'c'", "'a'", "'b'"},
+                "start": {"'c'", "'a'", "'b'"},
+            },
+        )
+
+    def test_repeat_with_separator(self) -> None:
+        grammar = """
+        start: ','.thing+ NEWLINE
+        thing: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {"NUMBER"}},
+        )
+
+    def test_optional_operator(self) -> None:
+        grammar = """
+        start: sum NEWLINE
+        sum: (term)? 'b'
+        term: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "term": {"NUMBER"},
+                "sum": {"NUMBER", "'b'"},
+                "start": {"'b'", "NUMBER"},
+            },
+        )
+
+    def test_optional_literal(self) -> None:
+        grammar = """
+        start: sum NEWLINE
+        sum: '+' ? term
+        term: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "term": {"NUMBER"},
+                "sum": {"'+'", "NUMBER"},
+                "start": {"'+'", "NUMBER"},
+            },
+        )
+
+    def test_optional_after(self) -> None:
+        grammar = """
+        start: term NEWLINE
+        term: NUMBER ['+']
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"term": {"NUMBER"}, "start": {"NUMBER"}},
+        )
+
+    def test_optional_before(self) -> None:
+        grammar = """
+        start: term NEWLINE
+        term: ['+'] NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"term": {"NUMBER", "'+'"}, "start": {"NUMBER", "'+'"}},
+        )
+
+    def test_repeat_0(self) -> None:
+        grammar = """
+        start: thing* "+" NEWLINE
+        thing: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {'"+"', "NUMBER"}},
+        )
+
+    def test_repeat_0_with_group(self) -> None:
+        grammar = """
+        start: ('+' '-')* term NEWLINE
+        term: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"term": {"NUMBER"}, "start": {"'+'", "NUMBER"}},
+        )
+
+    def test_repeat_1(self) -> None:
+        grammar = """
+        start: thing+ '-' NEWLINE
+        thing: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {"NUMBER"}},
+        )
+
+    def test_repeat_1_with_group(self) -> None:
+        grammar = """
+        start: ('+' term)+ term NEWLINE
+        term: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'"}}
+        )
+
+    def test_gather(self) -> None:
+        grammar = """
+        start: ','.thing+ NEWLINE
+        thing: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {"NUMBER"}},
+        )
+
+    def test_positive_lookahead(self) -> None:
+        grammar = """
+        start: expr NEWLINE
+        expr: &'a' opt
+        opt: 'a' | 'b' | 'c'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "expr": {"'a'"},
+                "start": {"'a'"},
+                "opt": {"'b'", "'c'", "'a'"},
+            },
+        )
+
+    def test_negative_lookahead(self) -> None:
+        grammar = """
+        start: expr NEWLINE
+        expr: !'a' opt
+        opt: 'a' | 'b' | 'c'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "opt": {"'b'", "'a'", "'c'"},
+                "expr": {"'b'", "'c'"},
+                "start": {"'b'", "'c'"},
+            },
+        )
+
+    def test_left_recursion(self) -> None:
+        grammar = """
+        start: expr NEWLINE
+        expr: ('-' term | expr '+' term | term)
+        term: NUMBER
+        foo: 'foo'
+        bar: 'bar'
+        baz: 'baz'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "expr": {"NUMBER", "'-'"},
+                "term": {"NUMBER"},
+                "start": {"NUMBER", "'-'"},
+                "foo": {"'foo'"},
+                "bar": {"'bar'"},
+                "baz": {"'baz'"},
+            },
+        )
+
+    def test_advance_left_recursion(self) -> None:
+        grammar = """
+        start: NUMBER | sign start
+        sign: ['-']
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"sign": {"'-'", ""}, "start": {"'-'", "NUMBER"}},
+        )
+
+    def test_mutual_left_recursion(self) -> None:
+        grammar = """
+        start: foo 'E'
+        foo: bar 'A' | 'B'
+        bar: foo 'C' | 'D'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "foo": {"'D'", "'B'"},
+                "bar": {"'D'"},
+                "start": {"'D'", "'B'"},
+            },
+        )
+
+    def test_nasty_left_recursion(self) -> None:
+        # TODO: Validate this
+        grammar = """
+        start: target '='
+        target: maybe '+' | NAME
+        maybe: maybe '-' | target
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"maybe": set(), "target": {"NAME"}, "start": {"NAME"}},
+        )
+
+    def test_nullable_rule(self) -> None:
+        grammar = """
+        start: sign thing $
+        sign: ['-']
+        thing: NUMBER
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "sign": {"", "'-'"},
+                "thing": {"NUMBER"},
+                "start": {"NUMBER", "'-'"},
+            },
+        )
+
+    def test_epsilon_production_in_start_rule(self) -> None:
+        grammar = """
+        start: ['-'] $
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar), {"start": {"ENDMARKER", "'-'"}}
+        )
+
+    def test_multiple_nullable_rules(self) -> None:
+        grammar = """
+        start: sign thing other another $
+        sign: ['-']
+        thing: ['+']
+        other: '*'
+        another: '/'
+        """
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "sign": {"", "'-'"},
+                "thing": {"'+'", ""},
+                "start": {"'+'", "'-'", "'*'"},
+                "other": {"'*'"},
+                "another": {"'/'"},
+            },
+        )

Dependencies/Python/Lib/test/test_peg_generator/test_grammar_validator.py

@@ -0,0 +1,51 @@
+import unittest
+from test import test_tools
+
+test_tools.skip_if_missing("peg_generator")
+with test_tools.imports_under_tool("peg_generator"):
+    from pegen.grammar_parser import GeneratedParser as GrammarParser
+    from pegen.validator import SubRuleValidator, ValidationError
+    from pegen.testutil import parse_string
+    from pegen.grammar import Grammar
+
+
+class TestPegen(unittest.TestCase):
+    def test_rule_with_no_collision(self) -> None:
+        grammar_source = """
+        start: bad_rule
+        sum:
+            | NAME '-' NAME
+            | NAME '+' NAME
+        """
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        validator = SubRuleValidator(grammar)
+        for rule_name, rule in grammar.rules.items():
+            validator.validate_rule(rule_name, rule)
+
+    def test_rule_with_simple_collision(self) -> None:
+        grammar_source = """
+        start: bad_rule
+        sum:
+            | NAME '+' NAME
+            | NAME '+' NAME ';'
+        """
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        validator = SubRuleValidator(grammar)
+        with self.assertRaises(ValidationError):
+            for rule_name, rule in grammar.rules.items():
+                validator.validate_rule(rule_name, rule)
+
+    def test_rule_with_collision_after_some_other_rules(self) -> None:
+        grammar_source = """
+        start: bad_rule
+        sum:
+            | NAME '+' NAME
+            | NAME '*' NAME ';'
+            | NAME '-' NAME
+            | NAME '+' NAME ';'
+        """
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        validator = SubRuleValidator(grammar)
+        with self.assertRaises(ValidationError):
+            for rule_name, rule in grammar.rules.items():
+                validator.validate_rule(rule_name, rule)