Update to Pygments 2.17.0.

Author: jmzambon

codehighlighter/description.xml

@@ -6,7 +6,7 @@
 
  <identifier value="javahelps.codehighlighter"/>
 
- <version value="2.5.2"/>
+ <version value="2.5.3"/>
 
  <update-information>
  <src xlink:href="https://raw.githubusercontent.com/jmzambon/libreoffice-code-highlighter/master/codehighlighter.update.xml" />

codehighlighter/python/pythonpath/pygments/__init__.py

@@ -26,7 +26,7 @@
 """
 from io import StringIO, BytesIO
 
-__version__ = '2.16.1'
+__version__ = '2.17.0'
 __docformat__ = 'restructuredtext'
 
 __all__ = ['lex', 'format', 'highlight']

codehighlighter/python/pythonpath/pygments/formatters/__init__.py

@@ -131,7 +131,7 @@ def get_formatter_for_filename(fn, **options):
  if name not in _formatter_cache:
  _load_formatters(modname)
  return _formatter_cache[name](**options)
- for cls in find_plugin_formatters():
+ for _name, cls in find_plugin_formatters():
  for filename in cls.filenames:
  if _fn_matches(fn, filename):
  return cls(**options)

codehighlighter/python/pythonpath/pygments/formatters/html.py

@@ -323,6 +323,7 @@ class ``"special"`` (default: ``0``).
  If set to the path of a ctags file, wrap names in anchor tags that
  link to their definitions. `lineanchors` should be used, and the
  tags file should specify line numbers (see the `-n` option to ctags).
+ The tags file is assumed to be encoded in UTF-8.
 
  .. versionadded:: 1.6
 
@@ -908,7 +909,7 @@ def _format_lines(self, tokensource):
  def _lookup_ctag(self, token):
  entry = ctags.TagEntry()
  if self._ctags.find(entry, token.encode(), 0):
- return entry['file'], entry['lineNumber']
+ return entry['file'].decode(), entry['lineNumber']
  else:
  return None, None
 

codehighlighter/python/pythonpath/pygments/lexer.py

@@ -199,20 +199,9 @@ def analyse_text(text):
  it's the same as if the return values was ``0.0``.
  """
 
- def get_tokens(self, text, unfiltered=False):
- """
- This method is the basic interface of a lexer. It is called by
- the `highlight()` function. It must process the text and return an
- iterable of ``(tokentype, value)`` pairs from `text`.
+ def _preprocess_lexer_input(self, text):
+ """Apply preprocessing such as decoding the input, removing BOM and normalizing newlines."""
 
- Normally, you don't need to override this method. The default
- implementation processes the options recognized by all lexers
- (`stripnl`, `stripall` and so on), and then yields all tokens
- from `get_tokens_unprocessed()`, with the ``index`` dropped.
-
- If `unfiltered` is set to `True`, the filtering mechanism is
- bypassed even if filters are defined.
- """
  if not isinstance(text, str):
  if self.encoding == 'guess':
  text, _ = guess_decode(text)
@@ -255,6 +244,24 @@ def get_tokens(self, text, unfiltered=False):
  if self.ensurenl and not text.endswith('\n'):
  text += '\n'
 
+ return text
+
+ def get_tokens(self, text, unfiltered=False):
+ """
+ This method is the basic interface of a lexer. It is called by
+ the `highlight()` function. It must process the text and return an
+ iterable of ``(tokentype, value)`` pairs from `text`.
+
+ Normally, you don't need to override this method. The default
+ implementation processes the options recognized by all lexers
+ (`stripnl`, `stripall` and so on), and then yields all tokens
+ from `get_tokens_unprocessed()`, with the ``index`` dropped.
+
+ If `unfiltered` is set to `True`, the filtering mechanism is
+ bypassed even if filters are defined.
+ """
+ text = self._preprocess_lexer_input(text)
+
  def streamer():
  for _, t, v in self.get_tokens_unprocessed(text):
  yield t, v

codehighlighter/python/pythonpath/pygments/lexers/__init__.py

@@ -22,6 +22,7 @@
 COMPAT = {
  'Python3Lexer': 'PythonLexer',
  'Python3TracebackLexer': 'PythonTracebackLexer',
+ 'LeanLexer': 'Lean3Lexer',
 }
 
 __all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',

codehighlighter/python/pythonpath/pygments/lexers/_mapping.py

@@ -247,9 +247,10 @@
  'JsgfLexer': ('pygments.lexers.grammar_notation', 'JSGF', ('jsgf',), ('*.jsgf',), ('application/jsgf', 'application/x-jsgf', 'text/jsgf')),
  'JsonBareObjectLexer': ('pygments.lexers.data', 'JSONBareObject', (), (), ()),
  'JsonLdLexer': ('pygments.lexers.data', 'JSON-LD', ('jsonld', 'json-ld'), ('*.jsonld',), ('application/ld+json',)),
- 'JsonLexer': ('pygments.lexers.data', 'JSON', ('json', 'json-object'), ('*.json', 'Pipfile.lock'), ('application/json', 'application/json-object')),
+ 'JsonLexer': ('pygments.lexers.data', 'JSON', ('json', 'json-object'), ('*.json', '*.jsonl', '*.ndjson', 'Pipfile.lock'), ('application/json', 'application/json-object', 'application/x-ndjson', 'application/jsonl', 'application/json-seq')),
  'JsonnetLexer': ('pygments.lexers.jsonnet', 'Jsonnet', ('jsonnet',), ('*.jsonnet', '*.libsonnet'), ()),
  'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)),
+ 'JsxLexer': ('pygments.lexers.jsx', 'JSX', ('jsx', 'react'), ('*.jsx', '*.react'), ('text/jsx', 'text/typescript-jsx')),
  'JuliaConsoleLexer': ('pygments.lexers.julia', 'Julia console', ('jlcon', 'julia-repl'), (), ()),
  'JuliaLexer': ('pygments.lexers.julia', 'Julia', ('julia', 'jl'), ('*.jl',), ('text/x-julia', 'application/x-julia')),
  'JuttleLexer': ('pygments.lexers.javascript', 'Juttle', ('juttle',), ('*.juttle',), ('application/juttle', 'application/x-juttle', 'text/x-juttle', 'text/juttle')),
@@ -260,13 +261,16 @@
  'KokaLexer': ('pygments.lexers.haskell', 'Koka', ('koka',), ('*.kk', '*.kki'), ('text/x-koka',)),
  'KotlinLexer': ('pygments.lexers.jvm', 'Kotlin', ('kotlin',), ('*.kt', '*.kts'), ('text/x-kotlin',)),
  'KuinLexer': ('pygments.lexers.kuin', 'Kuin', ('kuin',), ('*.kn',), ()),
+ 'KustoLexer': ('pygments.lexers.kusto', 'Kusto', ('kql', 'kusto'), ('*.kql', '*.kusto', '.csl'), ()),
  'LSLLexer': ('pygments.lexers.scripting', 'LSL', ('lsl',), ('*.lsl',), ('text/x-lsl',)),
  'LassoCssLexer': ('pygments.lexers.templates', 'CSS+Lasso', ('css+lasso',), (), ('text/css+lasso',)),
  'LassoHtmlLexer': ('pygments.lexers.templates', 'HTML+Lasso', ('html+lasso',), (), ('text/html+lasso', 'application/x-httpd-lasso', 'application/x-httpd-lasso[89]')),
  'LassoJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Lasso', ('javascript+lasso', 'js+lasso'), (), ('application/x-javascript+lasso', 'text/x-javascript+lasso', 'text/javascript+lasso')),
  'LassoLexer': ('pygments.lexers.javascript', 'Lasso', ('lasso', 'lassoscript'), ('*.lasso', '*.lasso[89]'), ('text/x-lasso',)),
  'LassoXmlLexer': ('pygments.lexers.templates', 'XML+Lasso', ('xml+lasso',), (), ('application/xml+lasso',)),
- 'LeanLexer': ('pygments.lexers.theorem', 'Lean', ('lean',), ('*.lean',), ('text/x-lean',)),
+ 'LdaprcLexer': ('pygments.lexers.ldap', 'LDAP configuration file', ('ldapconf', 'ldaprc'), ('.ldaprc', 'ldaprc', 'ldap.conf'), ('text/x-ldapconf',)),
+ 'LdifLexer': ('pygments.lexers.ldap', 'LDIF', ('ldif',), ('*.ldif',), ('text/x-ldif',)),
+ 'Lean3Lexer': ('pygments.lexers.lean', 'Lean', ('lean', 'lean3'), ('*.lean',), ('text/x-lean', 'text/x-lean3')),
  'LessCssLexer': ('pygments.lexers.css', 'LessCss', ('less',), ('*.less',), ('text/x-less-css',)),
  'LighttpdConfLexer': ('pygments.lexers.configs', 'Lighttpd configuration file', ('lighttpd', 'lighty'), ('lighttpd.conf',), ('text/x-lighttpd-conf',)),
  'LilyPondLexer': ('pygments.lexers.lilypond', 'LilyPond', ('lilypond',), ('*.ly',), ()),
@@ -388,6 +392,7 @@
  'PromQLLexer': ('pygments.lexers.promql', 'PromQL', ('promql',), ('*.promql',), ()),
  'PropertiesLexer': ('pygments.lexers.configs', 'Properties', ('properties', 'jproperties'), ('*.properties',), ('text/x-java-properties',)),
  'ProtoBufLexer': ('pygments.lexers.dsls', 'Protocol Buffer', ('protobuf', 'proto'), ('*.proto',), ()),
+ 'PrqlLexer': ('pygments.lexers.prql', 'PRQL', ('prql',), ('*.prql',), ('application/prql', 'application/x-prql')),
  'PsyshConsoleLexer': ('pygments.lexers.php', 'PsySH console session for PHP', ('psysh',), (), ()),
  'PtxLexer': ('pygments.lexers.ptx', 'PTX', ('ptx',), ('*.ptx',), ('text/x-ptx',)),
  'PugLexer': ('pygments.lexers.html', 'Pug', ('pug', 'jade'), ('*.pug', '*.jade'), ('text/x-pug', 'text/x-jade')),
@@ -396,7 +401,7 @@
  'Python2Lexer': ('pygments.lexers.python', 'Python 2.x', ('python2', 'py2'), (), ('text/x-python2', 'application/x-python2')),
  'Python2TracebackLexer': ('pygments.lexers.python', 'Python 2.x Traceback', ('py2tb',), ('*.py2tb',), ('text/x-python2-traceback',)),
  'PythonConsoleLexer': ('pygments.lexers.python', 'Python console session', ('pycon',), (), ('text/x-python-doctest',)),
- 'PythonLexer': ('pygments.lexers.python', 'Python', ('python', 'py', 'sage', 'python3', 'py3'), ('*.py', '*.pyw', '*.pyi', '*.jy', '*.sage', '*.sc', 'SConstruct', 'SConscript', '*.bzl', 'BUCK', 'BUILD', 'BUILD.bazel', 'WORKSPACE', '*.tac'), ('text/x-python', 'application/x-python', 'text/x-python3', 'application/x-python3')),
+ 'PythonLexer': ('pygments.lexers.python', 'Python', ('python', 'py', 'sage', 'python3', 'py3', 'bazel', 'starlark'), ('*.py', '*.pyw', '*.pyi', '*.jy', '*.sage', '*.sc', 'SConstruct', 'SConscript', '*.bzl', 'BUCK', 'BUILD', 'BUILD.bazel', 'WORKSPACE', '*.tac'), ('text/x-python', 'application/x-python', 'text/x-python3', 'application/x-python3')),
  'PythonTracebackLexer': ('pygments.lexers.python', 'Python Traceback', ('pytb', 'py3tb'), ('*.pytb', '*.py3tb'), ('text/x-python-traceback', 'text/x-python3-traceback')),
  'PythonUL4Lexer': ('pygments.lexers.ul4', 'Python+UL4', ('py+ul4',), ('*.pyul4',), ()),
  'QBasicLexer': ('pygments.lexers.basic', 'QBasic', ('qbasic', 'basic'), ('*.BAS', '*.bas'), ('text/basic',)),
@@ -488,7 +493,7 @@
  'SystemdLexer': ('pygments.lexers.configs', 'Systemd', ('systemd',), ('*.service', '*.socket', '*.device', '*.mount', '*.automount', '*.swap', '*.target', '*.path', '*.timer', '*.slice', '*.scope'), ()),
  'TAPLexer': ('pygments.lexers.testing', 'TAP', ('tap',), ('*.tap',), ()),
  'TNTLexer': ('pygments.lexers.tnt', 'Typographic Number Theory', ('tnt',), ('*.tnt',), ()),
- 'TOMLLexer': ('pygments.lexers.configs', 'TOML', ('toml',), ('*.toml', 'Pipfile', 'poetry.lock'), ()),
+ 'TOMLLexer': ('pygments.lexers.configs', 'TOML', ('toml',), ('*.toml', 'Pipfile', 'poetry.lock'), ('application/toml',)),
  'Tads3Lexer': ('pygments.lexers.int_fiction', 'TADS 3', ('tads3',), ('*.t',), ()),
  'TalLexer': ('pygments.lexers.tal', 'Tal', ('tal', 'uxntal'), ('*.tal',), ('text/x-uxntal',)),
  'TasmLexer': ('pygments.lexers.asm', 'TASM', ('tasm',), ('*.asm', '*.ASM', '*.tasm'), ('text/x-tasm',)),
@@ -540,6 +545,9 @@
  'VerilogLexer': ('pygments.lexers.hdl', 'verilog', ('verilog', 'v'), ('*.v',), ('text/x-verilog',)),
  'VhdlLexer': ('pygments.lexers.hdl', 'vhdl', ('vhdl',), ('*.vhdl', '*.vhd'), ('text/x-vhdl',)),
  'VimLexer': ('pygments.lexers.textedit', 'VimL', ('vim',), ('*.vim', '.vimrc', '.exrc', '.gvimrc', '_vimrc', '_exrc', '_gvimrc', 'vimrc', 'gvimrc'), ('text/x-vim',)),
+ 'VisualPrologGrammarLexer': ('pygments.lexers.vip', 'Visual Prolog Grammar', ('visualprologgrammar',), ('*.vipgrm',), ()),
+ 'VisualPrologLexer': ('pygments.lexers.vip', 'Visual Prolog', ('visualprolog',), ('*.pro', '*.cl', '*.i', '*.pack', '*.ph'), ()),
+ 'VyperLexer': ('pygments.lexers.vyper', 'Vyper', ('vyper',), ('*.vy',), ()),
  'WDiffLexer': ('pygments.lexers.diff', 'WDiff', ('wdiff',), ('*.wdiff',), ()),
  'WatLexer': ('pygments.lexers.webassembly', 'WebAssembly', ('wast', 'wat'), ('*.wat', '*.wast'), ()),
  'WebIDLLexer': ('pygments.lexers.webidl', 'Web IDL', ('webidl',), ('*.webidl',), ()),

codehighlighter/python/pythonpath/pygments/lexers/algebra.py

@@ -25,7 +25,7 @@ class GAPLexer(RegexLexer):
  .. versionadded:: 2.0
  """
  name = 'GAP'
- url = 'http://www.gap-system.org'
+ url = 'https://www.gap-system.org'
  aliases = ['gap']
  filenames = ['*.g', '*.gd', '*.gi', '*.gap']