👌 IMPROVE: Parsing performance

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ exclude: >
 
 repos:
 
- - repo: git://github.com/pre-commit/pre-commit-hooks
+ - repo: https://github.com/pre-commit/pre-commit-hooks
  rev: v2.2.3
  hooks:
  - id: check-json

markdown_it/extensions/container/index.py

@@ -36,7 +36,7 @@ def container_func(state: StateBlock, startLine: int, endLine: int, silent: bool
 
  # Check out the first character quickly,
  # this should filter out most of non-containers
- if marker_char != charCodeAt(state.src, start):
+ if marker_char != state.srcCharCode[start]:
  return False
 
  # Check out the rest of the marker string
@@ -79,7 +79,7 @@ def container_func(state: StateBlock, startLine: int, endLine: int, silent: bool
  # test
  break
 
- if marker_char != charCodeAt(state.src, start):
+ if marker_char != state.srcCharCode[start]:
  continue
 
  if state.sCount[nextLine] - state.blkIndent >= 4:

markdown_it/extensions/deflist/index.py

@@ -1,6 +1,5 @@
 """Process definition lists."""
 from markdown_it import MarkdownIt
-from markdown_it.common.utils import charCodeAt
 from markdown_it.rules_block import StateBlock
 
 
@@ -16,7 +15,7 @@ def skipMarker(state: StateBlock, line: int):
  return -1
 
  # Check bullet
- marker = charCodeAt(state.src, start)
+ marker = state.srcCharCode[start]
  start += 1
  if marker != 0x7E and marker != 0x3A: # ~ :
  return -1
@@ -119,7 +118,7 @@ def deflist(state: StateBlock, startLine: int, endLine: int, silent: bool):
  )
 
  while pos < maximum:
- ch = charCodeAt(state.src, pos)
+ ch = state.srcCharCode[pos]
 
  if isSpace(ch):
  if ch == 0x09:

markdown_it/extensions/footnote/index.py

@@ -6,7 +6,7 @@
 from markdown_it.rules_inline import StateInline
 from markdown_it.rules_block import StateBlock
 from markdown_it.helpers import parseLinkLabel
-from markdown_it.common.utils import isSpace, charCodeAt
+from markdown_it.common.utils import isSpace
 
 
 def footnote_plugin(md: MarkdownIt):
@@ -43,23 +43,23 @@ def footnote_def(state: StateBlock, startLine: int, endLine: int, silent: bool):
  if start + 4 > maximum:
  return False
 
- if charCodeAt(state.src, start) != 0x5B: # /* [ */
+ if state.srcCharCode[start] != 0x5B: # /* [ */
  return False
- if charCodeAt(state.src, start + 1) != 0x5E: # /* ^ */
+ if state.srcCharCode[start + 1] != 0x5E: # /* ^ */
  return False
 
  pos = start + 2
  while pos < maximum:
- if charCodeAt(state.src, pos) == 0x20:
+ if state.srcCharCode[pos] == 0x20:
  return False
- if charCodeAt(state.src, pos) == 0x5D: # /* ] */
+ if state.srcCharCode[pos] == 0x5D: # /* ] */
  break
  pos += 1
 
  if pos == start + 2: # no empty footnote labels
  return False
  pos += 1
- if pos + 1 >= maximum or charCodeAt(state.src, pos) != 0x3A: # /* : */
+ if pos + 1 >= maximum or state.srcCharCode[pos] != 0x3A: # /* : */
  return False
  if silent:
  return True
@@ -87,7 +87,7 @@ def footnote_def(state: StateBlock, startLine: int, endLine: int, silent: bool):
  )
 
  while pos < maximum:
- ch = charCodeAt(state.src, pos)
+ ch = state.srcCharCode[pos]
 
  if isSpace(ch):
  if ch == 0x09:
@@ -136,9 +136,9 @@ def footnote_inline(state: StateInline, silent: bool):
 
  if start + 2 >= maximum:
  return False
- if charCodeAt(state.src, start) != 0x5E: # /* ^ */
+ if state.srcCharCode[start] != 0x5E: # /* ^ */
  return False
- if charCodeAt(state.src, start + 1) != 0x5B: # /* [ */
+ if state.srcCharCode[start + 1] != 0x5B: # /* [ */
  return False
 
  labelStart = start + 2
@@ -182,18 +182,18 @@ def footnote_ref(state: StateInline, silent: bool):
 
  if "footnotes" not in state.env or "refs" not in state.env["footnotes"]:
  return False
- if charCodeAt(state.src, start) != 0x5B: # /* [ */
+ if state.srcCharCode[start] != 0x5B: # /* [ */
  return False
- if charCodeAt(state.src, start + 1) != 0x5E: # /* ^ */
+ if state.srcCharCode[start + 1] != 0x5E: # /* ^ */
  return False
 
  pos = start + 2
  while pos < maximum:
- if charCodeAt(state.src, pos) == 0x20:
+ if state.srcCharCode[pos] == 0x20:
  return False
- if charCodeAt(state.src, pos) == 0x0A:
+ if state.srcCharCode[pos] == 0x0A:
  return False
- if charCodeAt(state.src, pos) == 0x5D: # /* ] */
+ if state.srcCharCode[pos] == 0x5D: # /* ] */
  break
  pos += 1
 

markdown_it/extensions/front_matter/index.py

@@ -29,7 +29,7 @@ def frontMatter(state: StateBlock, startLine: int, endLine: int, silent: bool):
 
  # Check out the first character of the first line quickly,
  # this should filter out non-front matter
- if startLine != 0 or marker_char != charCodeAt(state.src, 0):
+ if startLine != 0 or marker_char != state.srcCharCode[0]:
  return False
 
  # Check out the rest of the marker string
@@ -73,7 +73,7 @@ def frontMatter(state: StateBlock, startLine: int, endLine: int, silent: bool):
  # test
  break
 
- if marker_char != charCodeAt(state.src, start):
+ if marker_char != state.srcCharCode[start]:
  continue
 
  if state.sCount[nextLine] - state.blkIndent >= 4:

markdown_it/extensions/myst_blocks/index.py

@@ -2,7 +2,7 @@
 
 from markdown_it import MarkdownIt
 from markdown_it.rules_block import StateBlock
-from markdown_it.common.utils import charCodeAt, isSpace, escapeHtml
+from markdown_it.common.utils import isSpace, escapeHtml
 
 
 TARGET_PATTERN = re.compile(r"^\(([a-zA-Z0-9\|\@\<\>\*\.\/\_\-\+\:]{1,100})\)\=\s*$")
@@ -40,7 +40,7 @@ def line_comment(state: StateBlock, startLine: int, endLine: int, silent: bool):
  if state.sCount[startLine] - state.blkIndent >= 4:
  return False
 
- marker = charCodeAt(state.src, pos)
+ marker = state.srcCharCode[pos]
  pos += 1
 
  # Check block marker /* % */
@@ -70,7 +70,7 @@ def block_break(state: StateBlock, startLine: int, endLine: int, silent: bool):
  if state.sCount[startLine] - state.blkIndent >= 4:
  return False
 
- marker = charCodeAt(state.src, pos)
+ marker = state.srcCharCode[pos]
  pos += 1
 
  # Check block marker /* + */
@@ -81,7 +81,7 @@ def block_break(state: StateBlock, startLine: int, endLine: int, silent: bool):
 
  cnt = 1
  while pos < maximum:
- ch = charCodeAt(state.src, pos)
+ ch = state.srcCharCode[pos]
  if ch != marker and not isSpace(ch):
  break
  if ch == marker:

markdown_it/extensions/myst_role/index.py

@@ -2,7 +2,7 @@
 
 from markdown_it import MarkdownIt
 from markdown_it.rules_inline import StateInline
-from markdown_it.common.utils import charCodeAt, escapeHtml
+from markdown_it.common.utils import escapeHtml
 
 
 PATTERN = re.compile(r"^\{([a-zA-Z0-9\_\-\+\:]{1,36})\}(`+)(?!`)(.+?)(?<!`)\2(?!`)")
@@ -15,7 +15,7 @@ def myst_role_plugin(md: MarkdownIt):
 
 def myst_role(state: StateInline, silent: bool):
  try:
- if charCodeAt(state.src, state.pos - 1) == 0x5C: # /* \ */
+ if state.srcCharCode[state.pos - 1] == 0x5C: # /* \ */
  # escaped (this could be improved in the case of edge case '\\{')
  return False
  except IndexError:

markdown_it/helpers/parse_link_label.py

@@ -5,7 +5,6 @@
 returns the end of the label
 
 """
-from ..common.utils import charCodeAt
 
 
 def parseLinkLabel(state, start, disableNested=False):
@@ -18,7 +17,7 @@ def parseLinkLabel(state, start, disableNested=False):
  level = 1
 
  while state.pos < state.posMax:
- marker = charCodeAt(state.src, state.pos)
+ marker = state.srcCharCode[state.pos]
  if marker == 0x5D: # /* ] */)
  level -= 1
  if level == 0:

markdown_it/parser_block.py

@@ -92,10 +92,10 @@ def tokenize(
  line += 1
  state.line = line
 
- def parse(self, src: str, md, env, outTokens: List[Token]):
+ def parse(self, src: str, md, env, outTokens: List[Token], ords: List[int] = None):
  """Process input string and push block tokens into `outTokens`."""
  if not src:
  return
- state = StateBlock(src, md, env, outTokens)
+ state = StateBlock(src, md, env, outTokens, ords)
  self.tokenize(state, state.line, state.lineMax)
  return state.tokens

markdown_it/port.yaml

@@ -11,6 +11,10 @@
  this is generally the main difference between the codes,
  because in python you can't do e.g. `for {i=1;i<x;i++} {}`
  - Use python version of `charCodeAt`
+ - |
+ Reduce use of charCodeAt() by storing char codes in a srcCharCodes attribute for state
+ objects and sharing those whenever possible
+ This provides a significant performance boost
  - |
  Use python's built-in `html.escape` and `urlparse.quote` methods, as a replacement for
  the JS dependencies [mdurl](https://www.npmjs.com/package/mdurl)

markdown_it/rules_block/blockquote.py

@@ -2,7 +2,7 @@
 import logging
 
 from .state_block import StateBlock
-from ..common.utils import isSpace, charCodeAt
+from ..common.utils import isSpace
 
 LOGGER = logging.getLogger(__name__)
 
@@ -22,7 +22,7 @@ def blockquote(state: StateBlock, startLine: int, endLine: int, silent: bool):
  return False
 
  # check the block quote marker
- if charCodeAt(state.src, pos) != 0x3E: # /* > */
+ if state.srcCharCode[pos] != 0x3E: # /* > */
  pos += 1
  return False
  pos += 1
@@ -40,15 +40,15 @@ def blockquote(state: StateBlock, startLine: int, endLine: int, silent: bool):
  )
 
  # skip one optional space after '>'
- if charCodeAt(state.src, pos) == 0x20: # /* space */
+ if state.srcCharCode[pos] == 0x20: # /* space */
  # ' > test '
  # ^ -- position start of line here:
  pos += 1
  initial += 1
  offset += 1
  adjustTab = False
  spaceAfterMarker = True
- elif charCodeAt(state.src, pos) == 0x09: # /* tab */
+ elif state.srcCharCode[pos] == 0x09: # /* tab */
  spaceAfterMarker = True
 
  if (state.bsCount[startLine] + offset) % 4 == 3:
@@ -71,7 +71,7 @@ def blockquote(state: StateBlock, startLine: int, endLine: int, silent: bool):
  state.bMarks[startLine] = pos
 
  while pos < max:
- ch = charCodeAt(state.src, pos)
+ ch = state.srcCharCode[pos]
 
  if isSpace(ch):
  if ch == 0x09: # / tab /
@@ -147,9 +147,7 @@ def blockquote(state: StateBlock, startLine: int, endLine: int, silent: bool):
  # Case 1: line is not inside the blockquote, and this line is empty.
  break
 
- evaluatesTrue = (
- charCodeAt(state.src, pos) == 0x3E and not wasOutdented
- ) # /* > */
+ evaluatesTrue = state.srcCharCode[pos] == 0x3E and not wasOutdented # /* > */
  pos += 1
  if evaluatesTrue:
  # This line is inside the blockquote.
@@ -162,15 +160,15 @@ def blockquote(state: StateBlock, startLine: int, endLine: int, silent: bool):
  )
 
  # skip one optional space after '>'
- if charCodeAt(state.src, pos) == 0x20: # /* space */
+ if state.srcCharCode[pos] == 0x20: # /* space */
  # ' > test '
  # ^ -- position start of line here:
  pos += 1
  initial += 1
  offset += 1
  adjustTab = False
  spaceAfterMarker = True
- elif charCodeAt(state.src, pos) == 0x09: # /* tab */
+ elif state.srcCharCode[pos] == 0x09: # /* tab */
  spaceAfterMarker = True
 
  if (state.bsCount[nextLine] + offset) % 4 == 3:
@@ -193,7 +191,7 @@ def blockquote(state: StateBlock, startLine: int, endLine: int, silent: bool):
  state.bMarks[nextLine] = pos
 
  while pos < max:
- ch = charCodeAt(state.src, pos)
+ ch = state.srcCharCode[pos]
 
  if isSpace(ch):
  if ch == 0x09:

markdown_it/rules_block/fence.py

@@ -1,7 +1,7 @@
 # fences (``` lang, ~~~ lang)
 import logging
 
-from ..common.utils import charCodeAt, stripEscape
+from ..common.utils import stripEscape
 from .state_block import StateBlock
 
 LOGGER = logging.getLogger(__name__)
@@ -22,7 +22,7 @@ def fence(state: StateBlock, startLine: int, endLine: int, silent: bool):
  if pos + 3 > maximum:
  return False
 
- marker = charCodeAt(state.src, pos)
+ marker = state.srcCharCode[pos]
 
  # /* ~ */ /* ` */
  if marker != 0x7E and marker != 0x60:
@@ -68,7 +68,7 @@ def fence(state: StateBlock, startLine: int, endLine: int, silent: bool):
  # test
  break
 
- if charCodeAt(state.src, pos) != marker:
+ if state.srcCharCode[pos] != marker:
  continue
 
  if state.sCount[nextLine] - state.blkIndent >= 4: