bpo-29568: Disable any characters between two percents for escaped percent "%%" in the format string for classic string formatting. (GH-513)

Author: serhiy-storchaka

Lib/test/test_format.py

@@ -70,12 +70,34 @@ def testcommon(formatstr, args, output=None, limit=None, overflowok=False):
  testformat(b_format, b_args, b_output, limit, overflowok)
  testformat(ba_format, b_args, ba_output, limit, overflowok)
 
+def test_exc(formatstr, args, exception, excmsg):
+ try:
+ testformat(formatstr, args)
+ except exception as exc:
+ if str(exc) == excmsg:
+ if verbose:
+ print("yes")
+ else:
+ if verbose: print('no')
+ print('Unexpected ', exception, ':', repr(str(exc)))
+ except:
+ if verbose: print('no')
+ print('Unexpected exception')
+ raise
+ else:
+ raise TestFailed('did not get expected exception: %s' % excmsg)
+
+def test_exc_common(formatstr, args, exception, excmsg):
+ # test str and bytes
+ test_exc(formatstr, args, exception, excmsg)
+ test_exc(formatstr.encode('ascii'), args, exception, excmsg)
 
 class FormatTest(unittest.TestCase):
 
  def test_common_format(self):
  # test the format identifiers that work the same across
  # str, bytes, and bytearrays (integer, float, oct, hex)
+ testcommon("%%", (), "%")
  testcommon("%.1d", (1,), "1")
  testcommon("%.*d", (sys.maxsize,1), overflowok=True) # expect overflow
  testcommon("%.100d", (1,), '00000000000000000000000000000000000000'
@@ -246,6 +268,20 @@ def test_common_format(self):
  testcommon('%g', 1.1, '1.1')
  testcommon('%#g', 1.1, '1.10000')
 
+ if verbose:
+ print('Testing exceptions')
+ test_exc_common('%', (), ValueError, "incomplete format")
+ test_exc_common('% %s', 1, ValueError,
+ "unsupported format character '%' (0x25) at index 2")
+ test_exc_common('%d', '1', TypeError,
+ "%d format: a number is required, not str")
+ test_exc_common('%d', b'1', TypeError,
+ "%d format: a number is required, not bytes")
+ test_exc_common('%x', '1', TypeError,
+ "%x format: an integer is required, not str")
+ test_exc_common('%x', 3.14, TypeError,
+ "%x format: an integer is required, not float")
+
  def test_str_format(self):
  testformat("%r", "\u0378", "'\\u0378'") # non printable
  testformat("%a", "\u0378", "'\\u0378'") # non printable
@@ -255,29 +291,10 @@ def test_str_format(self):
  # Test exception for unknown format characters, etc.
  if verbose:
  print('Testing exceptions')
- def test_exc(formatstr, args, exception, excmsg):
- try:
- testformat(formatstr, args)
- except exception as exc:
- if str(exc) == excmsg:
- if verbose:
- print("yes")
- else:
- if verbose: print('no')
- print('Unexpected ', exception, ':', repr(str(exc)))
- except:
- if verbose: print('no')
- print('Unexpected exception')
- raise
- else:
- raise TestFailed('did not get expected exception: %s' % excmsg)
  test_exc('abc %b', 1, ValueError,
  "unsupported format character 'b' (0x62) at index 5")
  #test_exc(unicode('abc %\u3000','raw-unicode-escape'), 1, ValueError,
  # "unsupported format character '?' (0x3000) at index 5")
- test_exc('%d', '1', TypeError, "%d format: a number is required, not str")
- test_exc('%x', '1', TypeError, "%x format: an integer is required, not str")
- test_exc('%x', 3.14, TypeError, "%x format: an integer is required, not float")
  test_exc('%g', '1', TypeError, "must be real number, not str")
  test_exc('no format', '1', TypeError,
  "not all arguments converted during string formatting")
@@ -334,28 +351,6 @@ def __bytes__(self):
  # Test exception for unknown format characters, etc.
  if verbose:
  print('Testing exceptions')
- def test_exc(formatstr, args, exception, excmsg):
- try:
- testformat(formatstr, args)
- except exception as exc:
- if str(exc) == excmsg:
- if verbose:
- print("yes")
- else:
- if verbose: print('no')
- print('Unexpected ', exception, ':', repr(str(exc)))
- except:
- if verbose: print('no')
- print('Unexpected exception')
- raise
- else:
- raise TestFailed('did not get expected exception: %s' % excmsg)
- test_exc(b'%d', '1', TypeError,
- "%d format: a number is required, not str")
- test_exc(b'%d', b'1', TypeError,
- "%d format: a number is required, not bytes")
- test_exc(b'%x', 3.14, TypeError,
- "%x format: an integer is required, not float")
  test_exc(b'%g', '1', TypeError, "float argument required, not str")
  test_exc(b'%g', b'1', TypeError, "float argument required, not bytes")
  test_exc(b'no format', 7, TypeError,

Misc/NEWS

@@ -10,6 +10,9 @@ What's New in Python 3.7.0 alpha 1?
 Core and Builtins
 -----------------
 
+- bpo-29568: Escaped percent "%%" in the format string for classic string
+ formatting no longer allows any characters between two percents.
+
 - bpo-29714: Fix a regression that bytes format may fail when containing zero
  bytes inside.
 

Objects/bytesobject.c

@@ -650,6 +650,12 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
 #endif
 
  fmt++;
+ if (*fmt == '%') {
+ *res++ = '%';
+ fmt++;
+ fmtcnt--;
+ continue;
+ }
  if (*fmt == '(') {
  const char *keystart;
  Py_ssize_t keylen;
@@ -794,11 +800,9 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
  "incomplete format");
  goto error;
  }
- if (c != '%') {
- v = getnextarg(args, arglen, &argidx);
- if (v == NULL)
- goto error;
- }
+ v = getnextarg(args, arglen, &argidx);
+ if (v == NULL)
+ goto error;
 
  if (fmtcnt < 0) {
  /* last writer: disable writer overallocation */
@@ -808,10 +812,6 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
  sign = 0;
  fill = ' ';
  switch (c) {
- case '%':
- *res++ = '%';
- continue;
-
  case 'r':
  // %r is only for 2/3 code; 3 only code should use %a
  case 'a':
@@ -1017,7 +1017,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
  res += (width - len);
  }
 
- if (dict && (argidx < arglen) && c != '%') {
+ if (dict && (argidx < arglen)) {
  PyErr_SetString(PyExc_TypeError,
  "not all arguments converted during bytes formatting");
  Py_XDECREF(temp);

Objects/unicodeobject.c

@@ -14617,12 +14617,6 @@ unicode_format_arg_format(struct unicode_formatter_t *ctx,
  if (ctx->fmtcnt == 0)
  ctx->writer.overallocate = 0;
 
- if (arg->ch == '%') {
- if (_PyUnicodeWriter_WriteCharInline(writer, '%') < 0)
- return -1;
- return 1;
- }
-
  v = unicode_format_getnextarg(ctx);
  if (v == NULL)
  return -1;
@@ -14882,6 +14876,13 @@ unicode_format_arg(struct unicode_formatter_t *ctx)
  int ret;
 
  arg.ch = PyUnicode_READ(ctx->fmtkind, ctx->fmtdata, ctx->fmtpos);
+ if (arg.ch == '%') {
+ ctx->fmtpos++;
+ ctx->fmtcnt--;
+ if (_PyUnicodeWriter_WriteCharInline(&ctx->writer, '%') < 0)
+ return -1;
+ return 0;
+ }
  arg.flags = 0;
  arg.width = -1;
  arg.prec = -1;
@@ -14903,7 +14904,7 @@ unicode_format_arg(struct unicode_formatter_t *ctx)
  return -1;
  }
 
- if (ctx->dict && (ctx->argidx < ctx->arglen) && arg.ch != '%') {
+ if (ctx->dict && (ctx->argidx < ctx->arglen)) {
  PyErr_SetString(PyExc_TypeError,
  "not all arguments converted during string formatting");
  return -1;