gh-156894: Fix the position of syntax errors which cover a range (GH-156901)

The callers of _PyTokenizer_syntaxerror_known_range() pass columns in bytes,
but SyntaxError.offset and end_offset are columns in characters.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: serhiy-storchaka

Lib/test/test_exceptions.py

@@ -233,6 +233,23 @@ def test_error_offset_continuation_characters(self):
         check = self.check
         check('"\\\n"(1 for c in I,\\\n\\', 2, 2)
 
+    def testSyntaxErrorRange(self):
+        # gh-156894: the position was reported in bytes, not in characters,
+        # for the errors which cover a range
+        for source, offset, end_offset in [
+            ('abcd = 00010', 8, 11),
+            ('\u03b1\u03b2\u03b3\u03b4 = 00010', 8, 11),
+            ('a\u0301b\u0308c\u20d7d\u1ab0 = 00010', 12, 15),
+            ("abcd = ub'a'", 8, 10),
+            ("\u03b1\u03b2\u03b3\u03b4 = ub'a'", 8, 10),
+            ("a\u0301b\u0308c\u20d7d\u1ab0 = ub'a'", 12, 14),
+        ]:
+            with self.subTest(source=source):
+                with self.assertRaises(SyntaxError) as cm:
+                    compile(source, '<testcase>', 'exec')
+                self.assertEqual(cm.exception.offset, offset)
+                self.assertEqual(cm.exception.end_offset, end_offset)
+
     def testSyntaxErrorOffset(self):
         check = self.check
         check('def fact(x):\n\treturn x!\n', 2, 10)

Lib/test/test_tokenize.py

@@ -2566,7 +2566,7 @@ def test_tolerant_incompatible_prefix_position_after_non_ascii(self):
             self._get_tokens('bé )tf"2 ', extra_tokens=True)
         self.assertEqual(
             caught.exception.args,
-            ("'f' and 't' prefixes are incompatible", (1, 6)),
+            ("'f' and 't' prefixes are incompatible", (1, 5)),
         )
 
     def test_tolerant_fstring_closer_at_expression_entry_depth(self):

Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst

@@ -0,0 +1,2 @@
+Fix the position of syntax errors which cover a range if the line contains
+non-ASCII characters before the error.

Parser/tokenizer/helpers.c

@@ -9,6 +9,20 @@
 
 /* ############## ERRORS ############## */
 
+/* Convert a 1-based column in bytes into a 1-based column in characters.
+   The line is UTF-8 encoded, so it is enough to skip continuation bytes. */
+static int
+byte_col_to_char_col(const char *line, int byte_col)
+{
+    int char_col = 1;
+    for (int i = 0; i < byte_col - 1; i++) {
+        if ((line[i] & 0xC0) != 0x80) {
+            char_col++;
+        }
+    }
+    return char_col;
+}
+
 static int
 _syntaxerror_range(struct tok_state *tok, const char *format,
                    int col_offset, int end_col_offset,
@@ -35,9 +49,15 @@ _syntaxerror_range(struct tok_state *tok, const char *format,
     if (col_offset == -1) {
         col_offset = (int)PyUnicode_GET_LENGTH(errtext);
     }
+    else if (col_offset > 0) {
+        col_offset = byte_col_to_char_col(tok->line_start, col_offset);
+    }
     if (end_col_offset == -1) {
         end_col_offset = col_offset;
     }
+    else if (end_col_offset > 0) {
+        end_col_offset = byte_col_to_char_col(tok->line_start, end_col_offset);
+    }
 
     Py_ssize_t line_len = strcspn(tok->line_start, "\n");
     if (line_len != tok->cur - tok->line_start) {