| """ |
| pygments.lexers.markup |
| ~~~~~~~~~~~~~~~~~~~~~~ |
| |
| Lexers for non-HTML markup languages. |
| |
| :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS. |
| :license: BSD, see LICENSE for details. |
| """ |
| |
| import re |
| |
| from pygments.lexers.html import XmlLexer |
| from pygments.lexers.javascript import JavascriptLexer |
| from pygments.lexers.css import CssLexer |
| from pygments.lexers.lilypond import LilyPondLexer |
| from pygments.lexers.data import JsonLexer |
| |
| from pygments.lexer import RegexLexer, DelegatingLexer, include, bygroups, \ |
| using, this, do_insertions, default, words |
| from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ |
| Number, Punctuation, Generic, Other, Whitespace |
| from pygments.util import get_bool_opt, ClassNotFound |
| |
| __all__ = ['BBCodeLexer', 'MoinWikiLexer', 'RstLexer', 'TexLexer', 'GroffLexer', |
| 'MozPreprocHashLexer', 'MozPreprocPercentLexer', |
| 'MozPreprocXulLexer', 'MozPreprocJavascriptLexer', |
| 'MozPreprocCssLexer', 'MarkdownLexer', 'TiddlyWiki5Lexer', 'WikitextLexer'] |
| |
| |
| class BBCodeLexer(RegexLexer): |
| """ |
| A lexer that highlights BBCode(-like) syntax. |
| |
| .. versionadded:: 0.6 |
| """ |
| |
| name = 'BBCode' |
| aliases = ['bbcode'] |
| mimetypes = ['text/x-bbcode'] |
| |
| tokens = { |
| 'root': [ |
| (r'[^[]+', Text), |
| # tag/end tag begin |
| (r'\[/?\w+', Keyword, 'tag'), |
| # stray bracket |
| (r'\[', Text), |
| ], |
| 'tag': [ |
| (r'\s+', Text), |
| # attribute with value |
| (r'(\w+)(=)("?[^\s"\]]+"?)', |
| bygroups(Name.Attribute, Operator, String)), |
| # tag argument (a la [color=green]) |
| (r'(=)("?[^\s"\]]+"?)', |
| bygroups(Operator, String)), |
| # tag end |
| (r'\]', Keyword, '#pop'), |
| ], |
| } |
| |
| |
| class MoinWikiLexer(RegexLexer): |
| """ |
| For MoinMoin (and Trac) Wiki markup. |
| |
| .. versionadded:: 0.7 |
| """ |
| |
| name = 'MoinMoin/Trac Wiki markup' |
| aliases = ['trac-wiki', 'moin'] |
| filenames = [] |
| mimetypes = ['text/x-trac-wiki'] |
| flags = re.MULTILINE | re.IGNORECASE |
| |
| tokens = { |
| 'root': [ |
| (r'^#.*$', Comment), |
| (r'(!)(\S+)', bygroups(Keyword, Text)), # Ignore-next |
| # Titles |
| (r'^(=+)([^=]+)(=+)(\s*#.+)?$', |
| bygroups(Generic.Heading, using(this), Generic.Heading, String)), |
| # Literal code blocks, with optional shebang |
| (r'(\{\{\{)(\n#!.+)?', bygroups(Name.Builtin, Name.Namespace), 'codeblock'), |
| (r'(\'\'\'?|\|\||`|__|~~|\^|,,|::)', Comment), # Formatting |
| # Lists |
| (r'^( +)([.*-])( )', bygroups(Text, Name.Builtin, Text)), |
| (r'^( +)([a-z]{1,5}\.)( )', bygroups(Text, Name.Builtin, Text)), |
| # Other Formatting |
| (r'\[\[\w+.*?\]\]', Keyword), # Macro |
| (r'(\[[^\s\]]+)(\s+[^\]]+?)?(\])', |
| bygroups(Keyword, String, Keyword)), # Link |
| (r'^----+$', Keyword), # Horizontal rules |
| (r'[^\n\'\[{!_~^,|]+', Text), |
| (r'\n', Text), |
| (r'.', Text), |
| ], |
| 'codeblock': [ |
| (r'\}\}\}', Name.Builtin, '#pop'), |
| # these blocks are allowed to be nested in Trac, but not MoinMoin |
| (r'\{\{\{', Text, '#push'), |
| (r'[^{}]+', Comment.Preproc), # slurp boring text |
| (r'.', Comment.Preproc), # allow loose { or } |
| ], |
| } |
| |
| |
| class RstLexer(RegexLexer): |
| """ |
| For reStructuredText markup. |
| |
| .. versionadded:: 0.7 |
| |
| Additional options accepted: |
| |
| `handlecodeblocks` |
| Highlight the contents of ``.. sourcecode:: language``, |
| ``.. code:: language`` and ``.. code-block:: language`` |
| directives with a lexer for the given language (default: |
| ``True``). |
| |
| .. versionadded:: 0.8 |
| """ |
| name = 'reStructuredText' |
| url = 'https://docutils.sourceforge.io/rst.html' |
| aliases = ['restructuredtext', 'rst', 'rest'] |
| filenames = ['*.rst', '*.rest'] |
| mimetypes = ["text/x-rst", "text/prs.fallenstein.rst"] |
| flags = re.MULTILINE |
| |
| def _handle_sourcecode(self, match): |
| from pygments.lexers import get_lexer_by_name |
| |
| # section header |
| yield match.start(1), Punctuation, match.group(1) |
| yield match.start(2), Text, match.group(2) |
| yield match.start(3), Operator.Word, match.group(3) |
| yield match.start(4), Punctuation, match.group(4) |
| yield match.start(5), Text, match.group(5) |
| yield match.start(6), Keyword, match.group(6) |
| yield match.start(7), Text, match.group(7) |
| |
| # lookup lexer if wanted and existing |
| lexer = None |
| if self.handlecodeblocks: |
| try: |
| lexer = get_lexer_by_name(match.group(6).strip()) |
| except ClassNotFound: |
| pass |
| indention = match.group(8) |
| indention_size = len(indention) |
| code = (indention + match.group(9) + match.group(10) + match.group(11)) |
| |
| # no lexer for this language. handle it like it was a code block |
| if lexer is None: |
| yield match.start(8), String, code |
| return |
| |
| # highlight the lines with the lexer. |
| ins = [] |
| codelines = code.splitlines(True) |
| code = '' |
| for line in codelines: |
| if len(line) > indention_size: |
| ins.append((len(code), [(0, Text, line[:indention_size])])) |
| code += line[indention_size:] |
| else: |
| code += line |
| yield from do_insertions(ins, lexer.get_tokens_unprocessed(code)) |
| |
| # from docutils.parsers.rst.states |
| closers = '\'")]}>\u2019\u201d\xbb!?' |
| unicode_delimiters = '\u2010\u2011\u2012\u2013\u2014\u00a0' |
| end_string_suffix = (r'((?=$)|(?=[-/:.,; \n\x00%s%s]))' |
| % (re.escape(unicode_delimiters), |
| re.escape(closers))) |
| |
| tokens = { |
| 'root': [ |
| # Heading with overline |
| (r'^(=+|-+|`+|:+|\.+|\'+|"+|~+|\^+|_+|\*+|\++|#+)([ \t]*\n)' |
| r'(.+)(\n)(\1)(\n)', |
| bygroups(Generic.Heading, Text, Generic.Heading, |
| Text, Generic.Heading, Text)), |
| # Plain heading |
| (r'^(\S.*)(\n)(={3,}|-{3,}|`{3,}|:{3,}|\.{3,}|\'{3,}|"{3,}|' |
| r'~{3,}|\^{3,}|_{3,}|\*{3,}|\+{3,}|#{3,})(\n)', |
| bygroups(Generic.Heading, Text, Generic.Heading, Text)), |
| # Bulleted lists |
| (r'^(\s*)([-*+])( .+\n(?:\1 .+\n)*)', |
| bygroups(Text, Number, using(this, state='inline'))), |
| # Numbered lists |
| (r'^(\s*)([0-9#ivxlcmIVXLCM]+\.)( .+\n(?:\1 .+\n)*)', |
| bygroups(Text, Number, using(this, state='inline'))), |
| (r'^(\s*)(\(?[0-9#ivxlcmIVXLCM]+\))( .+\n(?:\1 .+\n)*)', |
| bygroups(Text, Number, using(this, state='inline'))), |
| # Numbered, but keep words at BOL from becoming lists |
| (r'^(\s*)([A-Z]+\.)( .+\n(?:\1 .+\n)+)', |
| bygroups(Text, Number, using(this, state='inline'))), |
| (r'^(\s*)(\(?[A-Za-z]+\))( .+\n(?:\1 .+\n)+)', |
| bygroups(Text, Number, using(this, state='inline'))), |
| # Line blocks |
| (r'^(\s*)(\|)( .+\n(?:\| .+\n)*)', |
| bygroups(Text, Operator, using(this, state='inline'))), |
| # Sourcecode directives |
| (r'^( *\.\.)(\s*)((?:source)?code(?:-block)?)(::)([ \t]*)([^\n]+)' |
| r'(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\8.*)?\n)+)', |
| _handle_sourcecode), |
| # A directive |
| (r'^( *\.\.)(\s*)([\w:-]+?)(::)(?:([ \t]*)(.*))', |
| bygroups(Punctuation, Text, Operator.Word, Punctuation, Text, |
| using(this, state='inline'))), |
| # A reference target |
| (r'^( *\.\.)(\s*)(_(?:[^:\\]|\\.)+:)(.*?)$', |
| bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))), |
| # A footnote/citation target |
| (r'^( *\.\.)(\s*)(\[.+\])(.*?)$', |
| bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))), |
| # A substitution def |
| (r'^( *\.\.)(\s*)(\|.+\|)(\s*)([\w:-]+?)(::)(?:([ \t]*)(.*))', |
| bygroups(Punctuation, Text, Name.Tag, Text, Operator.Word, |
| Punctuation, Text, using(this, state='inline'))), |
| # Comments |
| (r'^ *\.\..*(\n( +.*\n|\n)+)?', Comment.Preproc), |
| # Field list marker |
| (r'^( *)(:(?:\\\\|\\:|[^:\n])+:(?=\s))([ \t]*)', |
| bygroups(Text, Name.Class, Text)), |
| # Definition list |
| (r'^(\S.*(?<!::)\n)((?:(?: +.*)\n)+)', |
| bygroups(using(this, state='inline'), using(this, state='inline'))), |
| # Code blocks |
| (r'(::)(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\3.*)?\n)+)', |
| bygroups(String.Escape, Text, String, String, Text, String)), |
| include('inline'), |
| ], |
| 'inline': [ |
| (r'\\.', Text), # escape |
| (r'``', String, 'literal'), # code |
| (r'(`.+?)(<.+?>)(`__?)', # reference with inline target |
| bygroups(String, String.Interpol, String)), |
| (r'`.+?`__?', String), # reference |
| (r'(`.+?`)(:[a-zA-Z0-9:-]+?:)?', |
| bygroups(Name.Variable, Name.Attribute)), # role |
| (r'(:[a-zA-Z0-9:-]+?:)(`.+?`)', |
| bygroups(Name.Attribute, Name.Variable)), # role (content first) |
| (r'\*\*.+?\*\*', Generic.Strong), # Strong emphasis |
| (r'\*.+?\*', Generic.Emph), # Emphasis |
| (r'\[.*?\]_', String), # Footnote or citation |
| (r'<.+?>', Name.Tag), # Hyperlink |
| (r'[^\\\n\[*`:]+', Text), |
| (r'.', Text), |
| ], |
| 'literal': [ |
| (r'[^`]+', String), |
| (r'``' + end_string_suffix, String, '#pop'), |
| (r'`', String), |
| ] |
| } |
| |
| def __init__(self, **options): |
| self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True) |
| RegexLexer.__init__(self, **options) |
| |
| def analyse_text(text): |
| if text[:2] == '..' and text[2:3] != '.': |
| return 0.3 |
| p1 = text.find("\n") |
| p2 = text.find("\n", p1 + 1) |
| if (p2 > -1 and # has two lines |
| p1 * 2 + 1 == p2 and # they are the same length |
| text[p1+1] in '-=' and # the next line both starts and ends with |
| text[p1+1] == text[p2-1]): # ...a sufficiently high header |
| return 0.5 |
| |
| |
| class TexLexer(RegexLexer): |
| """ |
| Lexer for the TeX and LaTeX typesetting languages. |
| """ |
| |
| name = 'TeX' |
| aliases = ['tex', 'latex'] |
| filenames = ['*.tex', '*.aux', '*.toc'] |
| mimetypes = ['text/x-tex', 'text/x-latex'] |
| |
| tokens = { |
| 'general': [ |
| (r'%.*?\n', Comment), |
| (r'[{}]', Name.Builtin), |
| (r'[&_^]', Name.Builtin), |
| ], |
| 'root': [ |
| (r'\\\[', String.Backtick, 'displaymath'), |
| (r'\\\(', String, 'inlinemath'), |
| (r'\$\$', String.Backtick, 'displaymath'), |
| (r'\$', String, 'inlinemath'), |
| (r'\\([a-zA-Z]+|.)', Keyword, 'command'), |
| (r'\\$', Keyword), |
| include('general'), |
| (r'[^\\$%&_^{}]+', Text), |
| ], |
| 'math': [ |
| (r'\\([a-zA-Z]+|.)', Name.Variable), |
| include('general'), |
| (r'[0-9]+', Number), |
| (r'[-=!+*/()\[\]]', Operator), |
| (r'[^=!+*/()\[\]\\$%&_^{}0-9-]+', Name.Builtin), |
| ], |
| 'inlinemath': [ |
| (r'\\\)', String, '#pop'), |
| (r'\$', String, '#pop'), |
| include('math'), |
| ], |
| 'displaymath': [ |
| (r'\\\]', String, '#pop'), |
| (r'\$\$', String, '#pop'), |
| (r'\$', Name.Builtin), |
| include('math'), |
| ], |
| 'command': [ |
| (r'\[.*?\]', Name.Attribute), |
| (r'\*', Keyword), |
| default('#pop'), |
| ], |
| } |
| |
| def analyse_text(text): |
| for start in ("\\documentclass", "\\input", "\\documentstyle", |
| "\\relax"): |
| if text[:len(start)] == start: |
| return True |
| |
| |
| class GroffLexer(RegexLexer): |
| """ |
| Lexer for the (g)roff typesetting language, supporting groff |
| extensions. Mainly useful for highlighting manpage sources. |
| |
| .. versionadded:: 0.6 |
| """ |
| |
| name = 'Groff' |
| aliases = ['groff', 'nroff', 'man'] |
| filenames = ['*.[1-9]', '*.man', '*.1p', '*.3pm'] |
| mimetypes = ['application/x-troff', 'text/troff'] |
| |
| tokens = { |
| 'root': [ |
| (r'(\.)(\w+)', bygroups(Text, Keyword), 'request'), |
| (r'\.', Punctuation, 'request'), |
| # Regular characters, slurp till we find a backslash or newline |
| (r'[^\\\n]+', Text, 'textline'), |
| default('textline'), |
| ], |
| 'textline': [ |
| include('escapes'), |
| (r'[^\\\n]+', Text), |
| (r'\n', Text, '#pop'), |
| ], |
| 'escapes': [ |
| # groff has many ways to write escapes. |
| (r'\\"[^\n]*', Comment), |
| (r'\\[fn]\w', String.Escape), |
| (r'\\\(.{2}', String.Escape), |
| (r'\\.\[.*\]', String.Escape), |
| (r'\\.', String.Escape), |
| (r'\\\n', Text, 'request'), |
| ], |
| 'request': [ |
| (r'\n', Text, '#pop'), |
| include('escapes'), |
| (r'"[^\n"]+"', String.Double), |
| (r'\d+', Number), |
| (r'\S+', String), |
| (r'\s+', Text), |
| ], |
| } |
| |
| def analyse_text(text): |
| if text[:1] != '.': |
| return False |
| if text[:3] == '.\\"': |
| return True |
| if text[:4] == '.TH ': |
| return True |
| if text[1:3].isalnum() and text[3].isspace(): |
| return 0.9 |
| |
| |
| class MozPreprocHashLexer(RegexLexer): |
| """ |
| Lexer for Mozilla Preprocessor files (with '#' as the marker). |
| |
| Other data is left untouched. |
| |
| .. versionadded:: 2.0 |
| """ |
| name = 'mozhashpreproc' |
| aliases = [name] |
| filenames = [] |
| mimetypes = [] |
| |
| tokens = { |
| 'root': [ |
| (r'^#', Comment.Preproc, ('expr', 'exprstart')), |
| (r'.+', Other), |
| ], |
| 'exprstart': [ |
| (r'(literal)(.*)', bygroups(Comment.Preproc, Text), '#pop:2'), |
| (words(( |
| 'define', 'undef', 'if', 'ifdef', 'ifndef', 'else', 'elif', |
| 'elifdef', 'elifndef', 'endif', 'expand', 'filter', 'unfilter', |
| 'include', 'includesubst', 'error')), |
| Comment.Preproc, '#pop'), |
| ], |
| 'expr': [ |
| (words(('!', '!=', '==', '&&', '||')), Operator), |
| (r'(defined)(\()', bygroups(Keyword, Punctuation)), |
| (r'\)', Punctuation), |
| (r'[0-9]+', Number.Decimal), |
| (r'__\w+?__', Name.Variable), |
| (r'@\w+?@', Name.Class), |
| (r'\w+', Name), |
| (r'\n', Text, '#pop'), |
| (r'\s+', Text), |
| (r'\S', Punctuation), |
| ], |
| } |
| |
| |
| class MozPreprocPercentLexer(MozPreprocHashLexer): |
| """ |
| Lexer for Mozilla Preprocessor files (with '%' as the marker). |
| |
| Other data is left untouched. |
| |
| .. versionadded:: 2.0 |
| """ |
| name = 'mozpercentpreproc' |
| aliases = [name] |
| filenames = [] |
| mimetypes = [] |
| |
| tokens = { |
| 'root': [ |
| (r'^%', Comment.Preproc, ('expr', 'exprstart')), |
| (r'.+', Other), |
| ], |
| } |
| |
| |
| class MozPreprocXulLexer(DelegatingLexer): |
| """ |
| Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the |
| `XmlLexer`. |
| |
| .. versionadded:: 2.0 |
| """ |
| name = "XUL+mozpreproc" |
| aliases = ['xul+mozpreproc'] |
| filenames = ['*.xul.in'] |
| mimetypes = [] |
| |
| def __init__(self, **options): |
| super().__init__(XmlLexer, MozPreprocHashLexer, **options) |
| |
| |
| class MozPreprocJavascriptLexer(DelegatingLexer): |
| """ |
| Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the |
| `JavascriptLexer`. |
| |
| .. versionadded:: 2.0 |
| """ |
| name = "Javascript+mozpreproc" |
| aliases = ['javascript+mozpreproc'] |
| filenames = ['*.js.in'] |
| mimetypes = [] |
| |
| def __init__(self, **options): |
| super().__init__(JavascriptLexer, MozPreprocHashLexer, **options) |
| |
| |
| class MozPreprocCssLexer(DelegatingLexer): |
| """ |
| Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the |
| `CssLexer`. |
| |
| .. versionadded:: 2.0 |
| """ |
| name = "CSS+mozpreproc" |
| aliases = ['css+mozpreproc'] |
| filenames = ['*.css.in'] |
| mimetypes = [] |
| |
| def __init__(self, **options): |
| super().__init__(CssLexer, MozPreprocPercentLexer, **options) |
| |
| |
| class MarkdownLexer(RegexLexer): |
| """ |
| For Markdown markup. |
| |
| .. versionadded:: 2.2 |
| """ |
| name = 'Markdown' |
| url = 'https://daringfireball.net/projects/markdown/' |
| aliases = ['markdown', 'md'] |
| filenames = ['*.md', '*.markdown'] |
| mimetypes = ["text/x-markdown"] |
| flags = re.MULTILINE |
| |
| def _handle_codeblock(self, match): |
| from pygments.lexers import get_lexer_by_name |
| |
| yield match.start('initial'), String.Backtick, match.group('initial') |
| yield match.start('lang'), String.Backtick, match.group('lang') |
| if match.group('afterlang') is not None: |
| yield match.start('whitespace'), Whitespace, match.group('whitespace') |
| yield match.start('extra'), Text, match.group('extra') |
| yield match.start('newline'), Whitespace, match.group('newline') |
| |
| # lookup lexer if wanted and existing |
| lexer = None |
| if self.handlecodeblocks: |
| try: |
| lexer = get_lexer_by_name(match.group('lang').strip()) |
| except ClassNotFound: |
| pass |
| code = match.group('code') |
| # no lexer for this language. handle it like it was a code block |
| if lexer is None: |
| yield match.start('code'), String, code |
| else: |
| # FIXME: aren't the offsets wrong? |
| yield from do_insertions([], lexer.get_tokens_unprocessed(code)) |
| |
| yield match.start('terminator'), String.Backtick, match.group('terminator') |
| |
| tokens = { |
| 'root': [ |
| # heading with '#' prefix (atx-style) |
| (r'(^#[^#].+)(\n)', bygroups(Generic.Heading, Text)), |
| # subheading with '#' prefix (atx-style) |
| (r'(^#{2,6}[^#].+)(\n)', bygroups(Generic.Subheading, Text)), |
| # heading with '=' underlines (Setext-style) |
| (r'^(.+)(\n)(=+)(\n)', bygroups(Generic.Heading, Text, Generic.Heading, Text)), |
| # subheading with '-' underlines (Setext-style) |
| (r'^(.+)(\n)(-+)(\n)', bygroups(Generic.Subheading, Text, Generic.Subheading, Text)), |
| # task list |
| (r'^(\s*)([*-] )(\[[ xX]\])( .+\n)', |
| bygroups(Whitespace, Keyword, Keyword, using(this, state='inline'))), |
| # bulleted list |
| (r'^(\s*)([*-])(\s)(.+\n)', |
| bygroups(Whitespace, Keyword, Whitespace, using(this, state='inline'))), |
| # numbered list |
| (r'^(\s*)([0-9]+\.)( .+\n)', |
| bygroups(Whitespace, Keyword, using(this, state='inline'))), |
| # quote |
| (r'^(\s*>\s)(.+\n)', bygroups(Keyword, Generic.Emph)), |
| # code block fenced by 3 backticks |
| (r'^(\s*```\n[\w\W]*?^\s*```$\n)', String.Backtick), |
| # code block with language |
| # Some tools include extra stuff after the language name, just |
| # highlight that as text. For example: https://docs.enola.dev/use/execmd |
| (r'''(?x) |
| ^(?P<initial>\s*```) |
| (?P<lang>[\w\-]+) |
| (?P<afterlang> |
| (?P<whitespace>[^\S\n]+) |
| (?P<extra>.*))? |
| (?P<newline>\n) |
| (?P<code>(.|\n)*?) |
| (?P<terminator>^\s*```$\n) |
| ''', |
| _handle_codeblock), |
| |
| include('inline'), |
| ], |
| 'inline': [ |
| # escape |
| (r'\\.', Text), |
| # inline code |
| (r'([^`]?)(`[^`\n]+`)', bygroups(Text, String.Backtick)), |
| # warning: the following rules eat outer tags. |
| # eg. **foo _bar_ baz** => foo and baz are not recognized as bold |
| # bold fenced by '**' |
| (r'([^\*]?)(\*\*[^* \n][^*\n]*\*\*)', bygroups(Text, Generic.Strong)), |
| # bold fenced by '__' |
| (r'([^_]?)(__[^_ \n][^_\n]*__)', bygroups(Text, Generic.Strong)), |
| # italics fenced by '*' |
| (r'([^\*]?)(\*[^* \n][^*\n]*\*)', bygroups(Text, Generic.Emph)), |
| # italics fenced by '_' |
| (r'([^_]?)(_[^_ \n][^_\n]*_)', bygroups(Text, Generic.Emph)), |
| # strikethrough |
| (r'([^~]?)(~~[^~ \n][^~\n]*~~)', bygroups(Text, Generic.Deleted)), |
| # mentions and topics (twitter and github stuff) |
| (r'[@#][\w/:]+', Name.Entity), |
| # (image?) links eg:  |
| (r'(!?\[)([^]]+)(\])(\()([^)]+)(\))', |
| bygroups(Text, Name.Tag, Text, Text, Name.Attribute, Text)), |
| # reference-style links, e.g.: |
| # [an example][id] |
| # [id]: http://example.com/ |
| (r'(\[)([^]]+)(\])(\[)([^]]*)(\])', |
| bygroups(Text, Name.Tag, Text, Text, Name.Label, Text)), |
| (r'^(\s*\[)([^]]*)(\]:\s*)(.+)', |
| bygroups(Text, Name.Label, Text, Name.Attribute)), |
| |
| # general text, must come last! |
| (r'[^\\\s]+', Text), |
| (r'.', Text), |
| ], |
| } |
| |
| def __init__(self, **options): |
| self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True) |
| RegexLexer.__init__(self, **options) |
| |
| |
| class TiddlyWiki5Lexer(RegexLexer): |
| """ |
| For TiddlyWiki5 markup. |
| |
| .. versionadded:: 2.7 |
| """ |
| name = 'tiddler' |
| url = 'https://tiddlywiki.com/#TiddlerFiles' |
| aliases = ['tid'] |
| filenames = ['*.tid'] |
| mimetypes = ["text/vnd.tiddlywiki"] |
| flags = re.MULTILINE |
| |
| def _handle_codeblock(self, match): |
| """ |
| match args: 1:backticks, 2:lang_name, 3:newline, 4:code, 5:backticks |
| """ |
| from pygments.lexers import get_lexer_by_name |
| |
| # section header |
| yield match.start(1), String, match.group(1) |
| yield match.start(2), String, match.group(2) |
| yield match.start(3), Text, match.group(3) |
| |
| # lookup lexer if wanted and existing |
| lexer = None |
| if self.handlecodeblocks: |
| try: |
| lexer = get_lexer_by_name(match.group(2).strip()) |
| except ClassNotFound: |
| pass |
| code = match.group(4) |
| |
| # no lexer for this language. handle it like it was a code block |
| if lexer is None: |
| yield match.start(4), String, code |
| return |
| |
| yield from do_insertions([], lexer.get_tokens_unprocessed(code)) |
| |
| yield match.start(5), String, match.group(5) |
| |
| def _handle_cssblock(self, match): |
| """ |
| match args: 1:style tag 2:newline, 3:code, 4:closing style tag |
| """ |
| from pygments.lexers import get_lexer_by_name |
| |
| # section header |
| yield match.start(1), String, match.group(1) |
| yield match.start(2), String, match.group(2) |
| |
| lexer = None |
| if self.handlecodeblocks: |
| try: |
| lexer = get_lexer_by_name('css') |
| except ClassNotFound: |
| pass |
| code = match.group(3) |
| |
| # no lexer for this language. handle it like it was a code block |
| if lexer is None: |
| yield match.start(3), String, code |
| return |
| |
| yield from do_insertions([], lexer.get_tokens_unprocessed(code)) |
| |
| yield match.start(4), String, match.group(4) |
| |
| tokens = { |
| 'root': [ |
| # title in metadata section |
| (r'^(title)(:\s)(.+\n)', bygroups(Keyword, Text, Generic.Heading)), |
| # headings |
| (r'^(!)([^!].+\n)', bygroups(Generic.Heading, Text)), |
| (r'^(!{2,6})(.+\n)', bygroups(Generic.Subheading, Text)), |
| # bulleted or numbered lists or single-line block quotes |
| # (can be mixed) |
| (r'^(\s*)([*#>]+)(\s*)(.+\n)', |
| bygroups(Text, Keyword, Text, using(this, state='inline'))), |
| # multi-line block quotes |
| (r'^(<<<.*\n)([\w\W]*?)(^<<<.*$)', bygroups(String, Text, String)), |
| # table header |
| (r'^(\|.*?\|h)$', bygroups(Generic.Strong)), |
| # table footer or caption |
| (r'^(\|.*?\|[cf])$', bygroups(Generic.Emph)), |
| # table class |
| (r'^(\|.*?\|k)$', bygroups(Name.Tag)), |
| # definitions |
| (r'^(;.*)$', bygroups(Generic.Strong)), |
| # text block |
| (r'^(```\n)([\w\W]*?)(^```$)', bygroups(String, Text, String)), |
| # code block with language |
| (r'^(```)(\w+)(\n)([\w\W]*?)(^```$)', _handle_codeblock), |
| # CSS style block |
| (r'^(<style>)(\n)([\w\W]*?)(^</style>$)', _handle_cssblock), |
| |
| include('keywords'), |
| include('inline'), |
| ], |
| 'keywords': [ |
| (words(( |
| '\\define', '\\end', 'caption', 'created', 'modified', 'tags', |
| 'title', 'type'), prefix=r'^', suffix=r'\b'), |
| Keyword), |
| ], |
| 'inline': [ |
| # escape |
| (r'\\.', Text), |
| # created or modified date |
| (r'\d{17}', Number.Integer), |
| # italics |
| (r'(\s)(//[^/]+//)((?=\W|\n))', |
| bygroups(Text, Generic.Emph, Text)), |
| # superscript |
| (r'(\s)(\^\^[^\^]+\^\^)', bygroups(Text, Generic.Emph)), |
| # subscript |
| (r'(\s)(,,[^,]+,,)', bygroups(Text, Generic.Emph)), |
| # underscore |
| (r'(\s)(__[^_]+__)', bygroups(Text, Generic.Strong)), |
| # bold |
| (r"(\s)(''[^']+'')((?=\W|\n))", |
| bygroups(Text, Generic.Strong, Text)), |
| # strikethrough |
| (r'(\s)(~~[^~]+~~)((?=\W|\n))', |
| bygroups(Text, Generic.Deleted, Text)), |
| # TiddlyWiki variables |
| (r'<<[^>]+>>', Name.Tag), |
| (r'\$\$[^$]+\$\$', Name.Tag), |
| (r'\$\([^)]+\)\$', Name.Tag), |
| # TiddlyWiki style or class |
| (r'^@@.*$', Name.Tag), |
| # HTML tags |
| (r'</?[^>]+>', Name.Tag), |
| # inline code |
| (r'`[^`]+`', String.Backtick), |
| # HTML escaped symbols |
| (r'&\S*?;', String.Regex), |
| # Wiki links |
| (r'(\[{2})([^]\|]+)(\]{2})', bygroups(Text, Name.Tag, Text)), |
| # External links |
| (r'(\[{2})([^]\|]+)(\|)([^]\|]+)(\]{2})', |
| bygroups(Text, Name.Tag, Text, Name.Attribute, Text)), |
| # Transclusion |
| (r'(\{{2})([^}]+)(\}{2})', bygroups(Text, Name.Tag, Text)), |
| # URLs |
| (r'(\b.?.?tps?://[^\s"]+)', bygroups(Name.Attribute)), |
| |
| # general text, must come last! |
| (r'[\w]+', Text), |
| (r'.', Text) |
| ], |
| } |
| |
| def __init__(self, **options): |
| self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True) |
| RegexLexer.__init__(self, **options) |
| |
| |
| class WikitextLexer(RegexLexer): |
| """ |
| For MediaWiki Wikitext. |
| |
| Parsing Wikitext is tricky, and results vary between different MediaWiki |
| installations, so we only highlight common syntaxes (built-in or from |
| popular extensions), and also assume templates produce no unbalanced |
| syntaxes. |
| |
| .. versionadded:: 2.15 |
| """ |
| name = 'Wikitext' |
| url = 'https://www.mediawiki.org/wiki/Wikitext' |
| aliases = ['wikitext', 'mediawiki'] |
| filenames = [] |
| mimetypes = ['text/x-wiki'] |
| flags = re.MULTILINE |
| |
| def nowiki_tag_rules(tag_name): |
| return [ |
| (r'(?i)(</)({})(\s*)(>)'.format(tag_name), bygroups(Punctuation, |
| Name.Tag, Whitespace, Punctuation), '#pop'), |
| include('entity'), |
| include('text'), |
| ] |
| |
| def plaintext_tag_rules(tag_name): |
| return [ |
| (r'(?si)(.*?)(</)({})(\s*)(>)'.format(tag_name), bygroups(Text, |
| Punctuation, Name.Tag, Whitespace, Punctuation), '#pop'), |
| ] |
| |
| def delegate_tag_rules(tag_name, lexer): |
| return [ |
| (r'(?i)(</)({})(\s*)(>)'.format(tag_name), bygroups(Punctuation, |
| Name.Tag, Whitespace, Punctuation), '#pop'), |
| (r'(?si).+?(?=</{}\s*>)'.format(tag_name), using(lexer)), |
| ] |
| |
| def text_rules(token): |
| return [ |
| (r'\w+', token), |
| (r'[^\S\n]+', token), |
| (r'(?s).', token), |
| ] |
| |
| def handle_syntaxhighlight(self, match, ctx): |
| from pygments.lexers import get_lexer_by_name |
| |
| attr_content = match.group() |
| start = 0 |
| index = 0 |
| while True: |
| index = attr_content.find('>', start) |
| # Exclude comment end (-->) |
| if attr_content[index-2:index] != '--': |
| break |
| start = index + 1 |
| |
| if index == -1: |
| # No tag end |
| yield from self.get_tokens_unprocessed(attr_content, stack=['root', 'attr']) |
| return |
| attr = attr_content[:index] |
| yield from self.get_tokens_unprocessed(attr, stack=['root', 'attr']) |
| yield match.start(3) + index, Punctuation, '>' |
| |
| lexer = None |
| content = attr_content[index+1:] |
| lang_match = re.findall(r'\blang=("|\'|)(\w+)(\1)', attr) |
| |
| if len(lang_match) >= 1: |
| # Pick the last match in case of multiple matches |
| lang = lang_match[-1][1] |
| try: |
| lexer = get_lexer_by_name(lang) |
| except ClassNotFound: |
| pass |
| |
| if lexer is None: |
| yield match.start() + index + 1, Text, content |
| else: |
| yield from lexer.get_tokens_unprocessed(content) |
| |
| def handle_score(self, match, ctx): |
| attr_content = match.group() |
| start = 0 |
| index = 0 |
| while True: |
| index = attr_content.find('>', start) |
| # Exclude comment end (-->) |
| if attr_content[index-2:index] != '--': |
| break |
| start = index + 1 |
| |
| if index == -1: |
| # No tag end |
| yield from self.get_tokens_unprocessed(attr_content, stack=['root', 'attr']) |
| return |
| attr = attr_content[:index] |
| content = attr_content[index+1:] |
| yield from self.get_tokens_unprocessed(attr, stack=['root', 'attr']) |
| yield match.start(3) + index, Punctuation, '>' |
| |
| lang_match = re.findall(r'\blang=("|\'|)(\w+)(\1)', attr) |
| # Pick the last match in case of multiple matches |
| lang = lang_match[-1][1] if len(lang_match) >= 1 else 'lilypond' |
| |
| if lang == 'lilypond': # Case sensitive |
| yield from LilyPondLexer().get_tokens_unprocessed(content) |
| else: # ABC |
| # FIXME: Use ABC lexer in the future |
| yield match.start() + index + 1, Text, content |
| |
| # a-z removed to prevent linter from complaining, REMEMBER to use (?i) |
| title_char = r' %!"$&\'()*,\-./0-9:;=?@A-Z\\\^_`~+\u0080-\uFFFF' |
| nbsp_char = r'(?:\t| |&\#0*160;|&\#[Xx]0*[Aa]0;|[ \xA0\u1680\u2000-\u200A\u202F\u205F\u3000])' |
| link_address = r'(?:[0-9.]+|\[[0-9a-f:.]+\]|[^\x00-\x20"<>\[\]\x7F\xA0\u1680\u2000-\u200A\u202F\u205F\u3000\uFFFD])' |
| link_char_class = r'[^\x00-\x20"<>\[\]\x7F\xA0\u1680\u2000-\u200A\u202F\u205F\u3000\uFFFD]' |
| double_slashes_i = { |
| '__FORCETOC__', '__NOCONTENTCONVERT__', '__NOCC__', '__NOEDITSECTION__', '__NOGALLERY__', |
| '__NOTITLECONVERT__', '__NOTC__', '__NOTOC__', '__TOC__', |
| } |
| double_slashes = { |
| '__EXPECTUNUSEDCATEGORY__', '__HIDDENCAT__', '__INDEX__', '__NEWSECTIONLINK__', |
| '__NOINDEX__', '__NONEWSECTIONLINK__', '__STATICREDIRECT__', '__NOGLOBAL__', |
| '__DISAMBIG__', '__EXPECTED_UNCONNECTED_PAGE__', |
| } |
| protocols = { |
| 'bitcoin:', 'ftp://', 'ftps://', 'geo:', 'git://', 'gopher://', 'http://', 'https://', |
| 'irc://', 'ircs://', 'magnet:', 'mailto:', 'mms://', 'news:', 'nntp://', 'redis://', |
| 'sftp://', 'sip:', 'sips:', 'sms:', 'ssh://', 'svn://', 'tel:', 'telnet://', 'urn:', |
| 'worldwind://', 'xmpp:', '//', |
| } |
| non_relative_protocols = protocols - {'//'} |
| html_tags = { |
| 'abbr', 'b', 'bdi', 'bdo', 'big', 'blockquote', 'br', 'caption', 'center', 'cite', 'code', |
| 'data', 'dd', 'del', 'dfn', 'div', 'dl', 'dt', 'em', 'font', 'h1', 'h2', 'h3', 'h4', 'h5', |
| 'h6', 'hr', 'i', 'ins', 'kbd', 'li', 'link', 'mark', 'meta', 'ol', 'p', 'q', 'rb', 'rp', |
| 'rt', 'rtc', 'ruby', 's', 'samp', 'small', 'span', 'strike', 'strong', 'sub', 'sup', |
| 'table', 'td', 'th', 'time', 'tr', 'tt', 'u', 'ul', 'var', 'wbr', |
| } |
| parser_tags = { |
| 'graph', 'charinsert', 'rss', 'chem', 'categorytree', 'nowiki', 'inputbox', 'math', |
| 'hiero', 'score', 'pre', 'ref', 'translate', 'imagemap', 'templatestyles', 'languages', |
| 'noinclude', 'mapframe', 'section', 'poem', 'syntaxhighlight', 'includeonly', 'tvar', |
| 'onlyinclude', 'templatedata', 'langconvert', 'timeline', 'dynamicpagelist', 'gallery', |
| 'maplink', 'ce', 'references', |
| } |
| variant_langs = { |
| # ZhConverter.php |
| 'zh', 'zh-hans', 'zh-hant', 'zh-cn', 'zh-hk', 'zh-mo', 'zh-my', 'zh-sg', 'zh-tw', |
| # UnConverter.php |
| 'uz', 'uz-latn', 'uz-cyrl', |
| # TlyConverter.php |
| 'tly', 'tly-cyrl', |
| # TgConverter.php |
| 'tg', 'tg-latn', |
| # SrConverter.php |
| 'sr', 'sr-ec', 'sr-el', |
| # ShiConverter.php |
| 'shi', 'shi-tfng', 'shi-latn', |
| # ShConverter.php |
| 'sh-latn', 'sh-cyrl', |
| # KuConverter.php |
| 'ku', 'ku-arab', 'ku-latn', |
| # KkConverter.php |
| 'kk', 'kk-cyrl', 'kk-latn', 'kk-arab', 'kk-kz', 'kk-tr', 'kk-cn', |
| # IuConverter.php |
| 'iu', 'ike-cans', 'ike-latn', |
| # GanConverter.php |
| 'gan', 'gan-hans', 'gan-hant', |
| # EnConverter.php |
| 'en', 'en-x-piglatin', |
| # CrhConverter.php |
| 'crh', 'crh-cyrl', 'crh-latn', |
| # BanConverter.php |
| 'ban', 'ban-bali', 'ban-x-dharma', 'ban-x-palmleaf', 'ban-x-pku', |
| } |
| magic_vars_i = { |
| 'ARTICLEPATH', 'INT', 'PAGEID', 'SCRIPTPATH', 'SERVER', 'SERVERNAME', 'STYLEPATH', |
| } |
| magic_vars = { |
| '!', '=', 'BASEPAGENAME', 'BASEPAGENAMEE', 'CASCADINGSOURCES', 'CONTENTLANGUAGE', |
| 'CONTENTLANG', 'CURRENTDAY', 'CURRENTDAY2', 'CURRENTDAYNAME', 'CURRENTDOW', 'CURRENTHOUR', |
| 'CURRENTMONTH', 'CURRENTMONTH2', 'CURRENTMONTH1', 'CURRENTMONTHABBREV', 'CURRENTMONTHNAME', |
| 'CURRENTMONTHNAMEGEN', 'CURRENTTIME', 'CURRENTTIMESTAMP', 'CURRENTVERSION', 'CURRENTWEEK', |
| 'CURRENTYEAR', 'DIRECTIONMARK', 'DIRMARK', 'FULLPAGENAME', 'FULLPAGENAMEE', 'LOCALDAY', |
| 'LOCALDAY2', 'LOCALDAYNAME', 'LOCALDOW', 'LOCALHOUR', 'LOCALMONTH', 'LOCALMONTH2', |
| 'LOCALMONTH1', 'LOCALMONTHABBREV', 'LOCALMONTHNAME', 'LOCALMONTHNAMEGEN', 'LOCALTIME', |
| 'LOCALTIMESTAMP', 'LOCALWEEK', 'LOCALYEAR', 'NAMESPACE', 'NAMESPACEE', 'NAMESPACENUMBER', |
| 'NUMBEROFACTIVEUSERS', 'NUMBEROFADMINS', 'NUMBEROFARTICLES', 'NUMBEROFEDITS', |
| 'NUMBEROFFILES', 'NUMBEROFPAGES', 'NUMBEROFUSERS', 'PAGELANGUAGE', 'PAGENAME', 'PAGENAMEE', |
| 'REVISIONDAY', 'REVISIONDAY2', 'REVISIONID', 'REVISIONMONTH', 'REVISIONMONTH1', |
| 'REVISIONSIZE', 'REVISIONTIMESTAMP', 'REVISIONUSER', 'REVISIONYEAR', 'ROOTPAGENAME', |
| 'ROOTPAGENAMEE', 'SITENAME', 'SUBJECTPAGENAME', 'ARTICLEPAGENAME', 'SUBJECTPAGENAMEE', |
| 'ARTICLEPAGENAMEE', 'SUBJECTSPACE', 'ARTICLESPACE', 'SUBJECTSPACEE', 'ARTICLESPACEE', |
| 'SUBPAGENAME', 'SUBPAGENAMEE', 'TALKPAGENAME', 'TALKPAGENAMEE', 'TALKSPACE', 'TALKSPACEE', |
| } |
| parser_functions_i = { |
| 'ANCHORENCODE', 'BIDI', 'CANONICALURL', 'CANONICALURLE', 'FILEPATH', 'FORMATNUM', |
| 'FULLURL', 'FULLURLE', 'GENDER', 'GRAMMAR', 'INT', r'\#LANGUAGE', 'LC', 'LCFIRST', 'LOCALURL', |
| 'LOCALURLE', 'NS', 'NSE', 'PADLEFT', 'PADRIGHT', 'PAGEID', 'PLURAL', 'UC', 'UCFIRST', |
| 'URLENCODE', |
| } |
| parser_functions = { |
| 'BASEPAGENAME', 'BASEPAGENAMEE', 'CASCADINGSOURCES', 'DEFAULTSORT', 'DEFAULTSORTKEY', |
| 'DEFAULTCATEGORYSORT', 'FULLPAGENAME', 'FULLPAGENAMEE', 'NAMESPACE', 'NAMESPACEE', |
| 'NAMESPACENUMBER', 'NUMBERINGROUP', 'NUMINGROUP', 'NUMBEROFACTIVEUSERS', 'NUMBEROFADMINS', |
| 'NUMBEROFARTICLES', 'NUMBEROFEDITS', 'NUMBEROFFILES', 'NUMBEROFPAGES', 'NUMBEROFUSERS', |
| 'PAGENAME', 'PAGENAMEE', 'PAGESINCATEGORY', 'PAGESINCAT', 'PAGESIZE', 'PROTECTIONEXPIRY', |
| 'PROTECTIONLEVEL', 'REVISIONDAY', 'REVISIONDAY2', 'REVISIONID', 'REVISIONMONTH', |
| 'REVISIONMONTH1', 'REVISIONTIMESTAMP', 'REVISIONUSER', 'REVISIONYEAR', 'ROOTPAGENAME', |
| 'ROOTPAGENAMEE', 'SUBJECTPAGENAME', 'ARTICLEPAGENAME', 'SUBJECTPAGENAMEE', |
| 'ARTICLEPAGENAMEE', 'SUBJECTSPACE', 'ARTICLESPACE', 'SUBJECTSPACEE', 'ARTICLESPACEE', |
| 'SUBPAGENAME', 'SUBPAGENAMEE', 'TALKPAGENAME', 'TALKPAGENAMEE', 'TALKSPACE', 'TALKSPACEE', |
| 'INT', 'DISPLAYTITLE', 'PAGESINNAMESPACE', 'PAGESINNS', |
| } |
| |
| tokens = { |
| 'root': [ |
| # Redirects |
| (r"""(?xi) |
| (\A\s*?)(\#REDIRECT:?) # may contain a colon |
| (\s+)(\[\[) (?=[^\]\n]* \]\]$) |
| """, |
| bygroups(Whitespace, Keyword, Whitespace, Punctuation), 'redirect-inner'), |
| # Subheadings |
| (r'^(={2,6})(.+?)(\1)(\s*$\n)', |
| bygroups(Generic.Subheading, Generic.Subheading, Generic.Subheading, Whitespace)), |
| # Headings |
| (r'^(=.+?=)(\s*$\n)', |
| bygroups(Generic.Heading, Whitespace)), |
| # Double-slashed magic words |
| (words(double_slashes_i, prefix=r'(?i)'), Name.Function.Magic), |
| (words(double_slashes), Name.Function.Magic), |
| # Raw URLs |
| (r'(?i)\b(?:{}){}{}*'.format('|'.join(protocols), |
| link_address, link_char_class), Name.Label), |
| # Magic links |
| (r'\b(?:RFC|PMID){}+[0-9]+\b'.format(nbsp_char), |
| Name.Function.Magic), |
| (r"""(?x) |
| \bISBN {nbsp_char} |
| (?: 97[89] {nbsp_dash}? )? |
| (?: [0-9] {nbsp_dash}? ){{9}} # escape format() |
| [0-9Xx]\b |
| """.format(nbsp_char=nbsp_char, nbsp_dash=f'(?:-|{nbsp_char})'), Name.Function.Magic), |
| include('list'), |
| include('inline'), |
| include('text'), |
| ], |
| 'redirect-inner': [ |
| (r'(\]\])(\s*?\n)', bygroups(Punctuation, Whitespace), '#pop'), |
| (r'(\#)([^#]*?)', bygroups(Punctuation, Name.Label)), |
| (r'(?i)[{}]+'.format(title_char), Name.Tag), |
| ], |
| 'list': [ |
| # Description lists |
| (r'^;', Keyword, 'dt'), |
| # Ordered lists, unordered lists and indents |
| (r'^[#:*]+', Keyword), |
| # Horizontal rules |
| (r'^-{4,}', Keyword), |
| ], |
| 'inline': [ |
| # Signatures |
| (r'~{3,5}', Keyword), |
| # Entities |
| include('entity'), |
| # Bold & italic |
| (r"('')(''')(?!')", bygroups(Generic.Emph, |
| Generic.EmphStrong), 'inline-italic-bold'), |
| (r"'''(?!')", Generic.Strong, 'inline-bold'), |
| (r"''(?!')", Generic.Emph, 'inline-italic'), |
| # Comments & parameters & templates |
| include('replaceable'), |
| # Media links |
| ( |
| r"""(?xi) |
| (\[\[) |
| (File|Image) (:) |
| ((?: [%s] | \{{2,3}[^{}]*?\}{2,3} | <!--[\s\S]*?--> )*) |
| (?: (\#) ([%s]*?) )? |
| """ % (title_char, f'{title_char}#'), |
| bygroups(Punctuation, Name.Namespace, Punctuation, |
| using(this, state=['wikilink-name']), Punctuation, Name.Label), |
| 'medialink-inner' |
| ), |
| # Wikilinks |
| ( |
| r"""(?xi) |
| (\[\[)(?!%s) # Should not contain URLs |
| (?: ([%s]*) (:))? |
| ((?: [%s] | \{{2,3}[^{}]*?\}{2,3} | <!--[\s\S]*?--> )*?) |
| (?: (\#) ([%s]*?) )? |
| (\]\]) |
| """ % ('|'.join(protocols), title_char.replace('/', ''), |
| title_char, f'{title_char}#'), |
| bygroups(Punctuation, Name.Namespace, Punctuation, |
| using(this, state=['wikilink-name']), Punctuation, Name.Label, Punctuation) |
| ), |
| ( |
| r"""(?xi) |
| (\[\[)(?!%s) |
| (?: ([%s]*) (:))? |
| ((?: [%s] | \{{2,3}[^{}]*?\}{2,3} | <!--[\s\S]*?--> )*?) |
| (?: (\#) ([%s]*?) )? |
| (\|) |
| """ % ('|'.join(protocols), title_char.replace('/', ''), |
| title_char, f'{title_char}#'), |
| bygroups(Punctuation, Name.Namespace, Punctuation, |
| using(this, state=['wikilink-name']), Punctuation, Name.Label, Punctuation), |
| 'wikilink-inner' |
| ), |
| # External links |
| ( |
| r"""(?xi) |
| (\[) |
| ((?:{}) {} {}*) |
| (\s*) |
| """.format('|'.join(protocols), link_address, link_char_class), |
| bygroups(Punctuation, Name.Label, Whitespace), |
| 'extlink-inner' |
| ), |
| # Tables |
| (r'^(:*)(\s*?)(\{\|)([^\n]*)$', bygroups(Keyword, |
| Whitespace, Punctuation, using(this, state=['root', 'attr'])), 'table'), |
| # HTML tags |
| (r'(?i)(<)({})\b'.format('|'.join(html_tags)), |
| bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'), |
| (r'(?i)(</)({})\b(\s*)(>)'.format('|'.join(html_tags)), |
| bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)), |
| # <nowiki> |
| (r'(?i)(<)(nowiki)\b', bygroups(Punctuation, |
| Name.Tag), ('tag-nowiki', 'tag-inner')), |
| # <pre> |
| (r'(?i)(<)(pre)\b', bygroups(Punctuation, |
| Name.Tag), ('tag-pre', 'tag-inner')), |
| # <categorytree> |
| (r'(?i)(<)(categorytree)\b', bygroups( |
| Punctuation, Name.Tag), ('tag-categorytree', 'tag-inner')), |
| # <hiero> |
| (r'(?i)(<)(hiero)\b', bygroups(Punctuation, |
| Name.Tag), ('tag-hiero', 'tag-inner')), |
| # <math> |
| (r'(?i)(<)(math)\b', bygroups(Punctuation, |
| Name.Tag), ('tag-math', 'tag-inner')), |
| # <chem> |
| (r'(?i)(<)(chem)\b', bygroups(Punctuation, |
| Name.Tag), ('tag-chem', 'tag-inner')), |
| # <ce> |
| (r'(?i)(<)(ce)\b', bygroups(Punctuation, |
| Name.Tag), ('tag-ce', 'tag-inner')), |
| # <charinsert> |
| (r'(?i)(<)(charinsert)\b', bygroups( |
| Punctuation, Name.Tag), ('tag-charinsert', 'tag-inner')), |
| # <templatedata> |
| (r'(?i)(<)(templatedata)\b', bygroups( |
| Punctuation, Name.Tag), ('tag-templatedata', 'tag-inner')), |
| # <gallery> |
| (r'(?i)(<)(gallery)\b', bygroups( |
| Punctuation, Name.Tag), ('tag-gallery', 'tag-inner')), |
| # <graph> |
| (r'(?i)(<)(gallery)\b', bygroups( |
| Punctuation, Name.Tag), ('tag-graph', 'tag-inner')), |
| # <dynamicpagelist> |
| (r'(?i)(<)(dynamicpagelist)\b', bygroups( |
| Punctuation, Name.Tag), ('tag-dynamicpagelist', 'tag-inner')), |
| # <inputbox> |
| (r'(?i)(<)(inputbox)\b', bygroups( |
| Punctuation, Name.Tag), ('tag-inputbox', 'tag-inner')), |
| # <rss> |
| (r'(?i)(<)(rss)\b', bygroups( |
| Punctuation, Name.Tag), ('tag-rss', 'tag-inner')), |
| # <imagemap> |
| (r'(?i)(<)(imagemap)\b', bygroups( |
| Punctuation, Name.Tag), ('tag-imagemap', 'tag-inner')), |
| # <syntaxhighlight> |
| (r'(?i)(</)(syntaxhighlight)\b(\s*)(>)', |
| bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)), |
| (r'(?si)(<)(syntaxhighlight)\b([^>]*?(?<!/)>.*?)(?=</\2\s*>)', |
| bygroups(Punctuation, Name.Tag, handle_syntaxhighlight)), |
| # <syntaxhighlight>: Fallback case for self-closing tags |
| (r'(?i)(<)(syntaxhighlight)\b(\s*?)((?:[^>]|-->)*?)(/\s*?(?<!--)>)', bygroups( |
| Punctuation, Name.Tag, Whitespace, using(this, state=['root', 'attr']), Punctuation)), |
| # <source> |
| (r'(?i)(</)(source)\b(\s*)(>)', |
| bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)), |
| (r'(?si)(<)(source)\b([^>]*?(?<!/)>.*?)(?=</\2\s*>)', |
| bygroups(Punctuation, Name.Tag, handle_syntaxhighlight)), |
| # <source>: Fallback case for self-closing tags |
| (r'(?i)(<)(source)\b(\s*?)((?:[^>]|-->)*?)(/\s*?(?<!--)>)', bygroups( |
| Punctuation, Name.Tag, Whitespace, using(this, state=['root', 'attr']), Punctuation)), |
| # <score> |
| (r'(?i)(</)(score)\b(\s*)(>)', |
| bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)), |
| (r'(?si)(<)(score)\b([^>]*?(?<!/)>.*?)(?=</\2\s*>)', |
| bygroups(Punctuation, Name.Tag, handle_score)), |
| # <score>: Fallback case for self-closing tags |
| (r'(?i)(<)(score)\b(\s*?)((?:[^>]|-->)*?)(/\s*?(?<!--)>)', bygroups( |
| Punctuation, Name.Tag, Whitespace, using(this, state=['root', 'attr']), Punctuation)), |
| # Other parser tags |
| (r'(?i)(<)({})\b'.format('|'.join(parser_tags)), |
| bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'), |
| (r'(?i)(</)({})\b(\s*)(>)'.format('|'.join(parser_tags)), |
| bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)), |
| # LanguageConverter markups |
| ( |
| r"""(?xi) |
| (-\{{) # Escape format() |
| (?: ([^|]) (\|))? |
| (?: (\s* (?:{variants}) \s*) (=>))? |
| (\s* (?:{variants}) \s*) (:) |
| """.format(variants='|'.join(variant_langs)), |
| bygroups(Punctuation, Keyword, Punctuation, |
| Name.Label, Operator, Name.Label, Punctuation), |
| 'lc-inner' |
| ), |
| (r'-\{(?!\{)', Punctuation, 'lc-raw'), |
| ], |
| 'wikilink-name': [ |
| include('replaceable'), |
| (r'[^{<]+', Name.Tag), |
| (r'(?s).', Name.Tag), |
| ], |
| 'wikilink-inner': [ |
| # Quit in case of another wikilink |
| (r'(?=\[\[)', Punctuation, '#pop'), |
| (r'\]\]', Punctuation, '#pop'), |
| include('inline'), |
| include('text'), |
| ], |
| 'medialink-inner': [ |
| (r'\]\]', Punctuation, '#pop'), |
| (r'(\|)([^\n=|]*)(=)', |
| bygroups(Punctuation, Name.Attribute, Operator)), |
| (r'\|', Punctuation), |
| include('inline'), |
| include('text'), |
| ], |
| 'quote-common': [ |
| # Quit in case of link/template endings |
| (r'(?=\]\]|\{\{|\}\})', Punctuation, '#pop'), |
| (r'\n', Text, '#pop'), |
| ], |
| 'inline-italic': [ |
| include('quote-common'), |
| (r"('')(''')(?!')", bygroups(Generic.Emph, |
| Generic.Strong), ('#pop', 'inline-bold')), |
| (r"'''(?!')", Generic.EmphStrong, ('#pop', 'inline-italic-bold')), |
| (r"''(?!')", Generic.Emph, '#pop'), |
| include('inline'), |
| include('text-italic'), |
| ], |
| 'inline-bold': [ |
| include('quote-common'), |
| (r"(''')('')(?!')", bygroups( |
| Generic.Strong, Generic.Emph), ('#pop', 'inline-italic')), |
| (r"'''(?!')", Generic.Strong, '#pop'), |
| (r"''(?!')", Generic.EmphStrong, ('#pop', 'inline-bold-italic')), |
| include('inline'), |
| include('text-bold'), |
| ], |
| 'inline-bold-italic': [ |
| include('quote-common'), |
| (r"('')(''')(?!')", bygroups(Generic.EmphStrong, |
| Generic.Strong), '#pop'), |
| (r"'''(?!')", Generic.EmphStrong, ('#pop', 'inline-italic')), |
| (r"''(?!')", Generic.EmphStrong, ('#pop', 'inline-bold')), |
| include('inline'), |
| include('text-bold-italic'), |
| ], |
| 'inline-italic-bold': [ |
| include('quote-common'), |
| (r"(''')('')(?!')", bygroups( |
| Generic.EmphStrong, Generic.Emph), '#pop'), |
| (r"'''(?!')", Generic.EmphStrong, ('#pop', 'inline-italic')), |
| (r"''(?!')", Generic.EmphStrong, ('#pop', 'inline-bold')), |
| include('inline'), |
| include('text-bold-italic'), |
| ], |
| 'lc-inner': [ |
| ( |
| r"""(?xi) |
| (;) |
| (?: (\s* (?:{variants}) \s*) (=>))? |
| (\s* (?:{variants}) \s*) (:) |
| """.format(variants='|'.join(variant_langs)), |
| bygroups(Punctuation, Name.Label, |
| Operator, Name.Label, Punctuation) |
| ), |
| (r';?\s*?\}-', Punctuation, '#pop'), |
| include('inline'), |
| include('text'), |
| ], |
| 'lc-raw': [ |
| (r'\}-', Punctuation, '#pop'), |
| include('inline'), |
| include('text'), |
| ], |
| 'replaceable': [ |
| # Comments |
| (r'<!--[\s\S]*?(?:-->|\Z)', Comment.Multiline), |
| # Parameters |
| ( |
| r"""(?x) |
| (\{{3}) |
| ([^|]*?) |
| (?=\}{3}|\|) |
| """, |
| bygroups(Punctuation, Name.Variable), |
| 'parameter-inner', |
| ), |
| # Magic variables |
| (r'(?i)(\{\{)(\s*)(%s)(\s*)(\}\})' % '|'.join(magic_vars_i), |
| bygroups(Punctuation, Whitespace, Name.Function, Whitespace, Punctuation)), |
| (r'(\{\{)(\s*)(%s)(\s*)(\}\})' % '|'.join(magic_vars), |
| bygroups(Punctuation, Whitespace, Name.Function, Whitespace, Punctuation)), |
| # Parser functions & templates |
| (r'\{\{', Punctuation, 'template-begin-space'), |
| # <tvar> legacy syntax |
| (r'(?i)(<)(tvar)\b(\|)([^>]*?)(>)', bygroups(Punctuation, |
| Name.Tag, Punctuation, String, Punctuation)), |
| (r'</>', Punctuation, '#pop'), |
| # <tvar> |
| (r'(?i)(<)(tvar)\b', bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'), |
| (r'(?i)(</)(tvar)\b(\s*)(>)', |
| bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)), |
| ], |
| 'parameter-inner': [ |
| (r'\}{3}', Punctuation, '#pop'), |
| (r'\|', Punctuation), |
| include('inline'), |
| include('text'), |
| ], |
| 'template-begin-space': [ |
| # Templates allow line breaks at the beginning, and due to how MediaWiki handles |
| # comments, an extra state is required to handle things like {{\n<!---->\n name}} |
| (r'<!--[\s\S]*?(?:-->|\Z)', Comment.Multiline), |
| (r'\s+', Whitespace), |
| # Parser functions |
| ( |
| r'(?i)(\#[%s]*?|%s)(:)' % (title_char, |
| '|'.join(parser_functions_i)), |
| bygroups(Name.Function, Punctuation), ('#pop', 'template-inner') |
| ), |
| ( |
| r'(%s)(:)' % ('|'.join(parser_functions)), |
| bygroups(Name.Function, Punctuation), ('#pop', 'template-inner') |
| ), |
| # Templates |
| ( |
| r'(?i)([%s]*?)(:)' % title_char, |
| bygroups(Name.Namespace, Punctuation), ('#pop', 'template-name') |
| ), |
| default(('#pop', 'template-name'),), |
| ], |
| 'template-name': [ |
| (r'(\s*?)(\|)', bygroups(Text, Punctuation), ('#pop', 'template-inner')), |
| (r'\}\}', Punctuation, '#pop'), |
| (r'\n', Text, '#pop'), |
| include('replaceable'), |
| *text_rules(Name.Tag), |
| ], |
| 'template-inner': [ |
| (r'\}\}', Punctuation, '#pop'), |
| (r'\|', Punctuation), |
| ( |
| r"""(?x) |
| (?<=\|) |
| ( (?: (?! \{\{ | \}\} )[^=\|<])*? ) # Exclude templates and tags |
| (=) |
| """, |
| bygroups(Name.Label, Operator) |
| ), |
| include('inline'), |
| include('text'), |
| ], |
| 'table': [ |
| # Use [ \t\n\r\0\x0B] instead of \s to follow PHP trim() behavior |
| # Endings |
| (r'^([ \t\n\r\0\x0B]*?)(\|\})', |
| bygroups(Whitespace, Punctuation), '#pop'), |
| # Table rows |
| (r'^([ \t\n\r\0\x0B]*?)(\|-+)(.*)$', bygroups(Whitespace, Punctuation, |
| using(this, state=['root', 'attr']))), |
| # Captions |
| ( |
| r"""(?x) |
| ^([ \t\n\r\0\x0B]*?)(\|\+) |
| # Exclude links, template and tags |
| (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|) )? |
| (.*?)$ |
| """, |
| bygroups(Whitespace, Punctuation, using(this, state=[ |
| 'root', 'attr']), Punctuation, Generic.Heading), |
| ), |
| # Table data |
| ( |
| r"""(?x) |
| ( ^(?:[ \t\n\r\0\x0B]*?)\| | \|\| ) |
| (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|)(?!\|) )? |
| """, |
| bygroups(Punctuation, using(this, state=[ |
| 'root', 'attr']), Punctuation), |
| ), |
| # Table headers |
| ( |
| r"""(?x) |
| ( ^(?:[ \t\n\r\0\x0B]*?)! ) |
| (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|)(?!\|) )? |
| """, |
| bygroups(Punctuation, using(this, state=[ |
| 'root', 'attr']), Punctuation), |
| 'table-header', |
| ), |
| include('list'), |
| include('inline'), |
| include('text'), |
| ], |
| 'table-header': [ |
| # Requires another state for || handling inside headers |
| (r'\n', Text, '#pop'), |
| ( |
| r"""(?x) |
| (!!|\|\|) |
| (?: |
| ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? ) |
| (\|)(?!\|) |
| )? |
| """, |
| bygroups(Punctuation, using(this, state=[ |
| 'root', 'attr']), Punctuation) |
| ), |
| *text_rules(Generic.Subheading), |
| ], |
| 'entity': [ |
| (r'&\S*?;', Name.Entity), |
| ], |
| 'dt': [ |
| (r'\n', Text, '#pop'), |
| include('inline'), |
| (r':', Keyword, '#pop'), |
| include('text'), |
| ], |
| 'extlink-inner': [ |
| (r'\]', Punctuation, '#pop'), |
| include('inline'), |
| include('text'), |
| ], |
| 'nowiki-ish': [ |
| include('entity'), |
| include('text'), |
| ], |
| 'attr': [ |
| include('replaceable'), |
| (r'\s+', Whitespace), |
| (r'(=)(\s*)(")', bygroups(Operator, Whitespace, String.Double), 'attr-val-2'), |
| (r"(=)(\s*)(')", bygroups(Operator, Whitespace, String.Single), 'attr-val-1'), |
| (r'(=)(\s*)', bygroups(Operator, Whitespace), 'attr-val-0'), |
| (r'[\w:-]+', Name.Attribute), |
| |
| ], |
| 'attr-val-0': [ |
| (r'\s', Whitespace, '#pop'), |
| include('replaceable'), |
| *text_rules(String), |
| ], |
| 'attr-val-1': [ |
| (r"'", String.Single, '#pop'), |
| include('replaceable'), |
| *text_rules(String.Single), |
| ], |
| 'attr-val-2': [ |
| (r'"', String.Double, '#pop'), |
| include('replaceable'), |
| *text_rules(String.Double), |
| ], |
| 'tag-inner-ordinary': [ |
| (r'/?\s*>', Punctuation, '#pop'), |
| include('tag-attr'), |
| ], |
| 'tag-inner': [ |
| # Return to root state for self-closing tags |
| (r'/\s*>', Punctuation, '#pop:2'), |
| (r'\s*>', Punctuation, '#pop'), |
| include('tag-attr'), |
| ], |
| # There states below are just like their non-tag variants, the key difference is |
| # they forcibly quit when encountering tag closing markup |
| 'tag-attr': [ |
| include('replaceable'), |
| (r'\s+', Whitespace), |
| (r'(=)(\s*)(")', bygroups(Operator, |
| Whitespace, String.Double), 'tag-attr-val-2'), |
| (r"(=)(\s*)(')", bygroups(Operator, |
| Whitespace, String.Single), 'tag-attr-val-1'), |
| (r'(=)(\s*)', bygroups(Operator, Whitespace), 'tag-attr-val-0'), |
| (r'[\w:-]+', Name.Attribute), |
| |
| ], |
| 'tag-attr-val-0': [ |
| (r'\s', Whitespace, '#pop'), |
| (r'/?>', Punctuation, '#pop:2'), |
| include('replaceable'), |
| *text_rules(String), |
| ], |
| 'tag-attr-val-1': [ |
| (r"'", String.Single, '#pop'), |
| (r'/?>', Punctuation, '#pop:2'), |
| include('replaceable'), |
| *text_rules(String.Single), |
| ], |
| 'tag-attr-val-2': [ |
| (r'"', String.Double, '#pop'), |
| (r'/?>', Punctuation, '#pop:2'), |
| include('replaceable'), |
| *text_rules(String.Double), |
| ], |
| 'tag-nowiki': nowiki_tag_rules('nowiki'), |
| 'tag-pre': nowiki_tag_rules('pre'), |
| 'tag-categorytree': plaintext_tag_rules('categorytree'), |
| 'tag-dynamicpagelist': plaintext_tag_rules('dynamicpagelist'), |
| 'tag-hiero': plaintext_tag_rules('hiero'), |
| 'tag-inputbox': plaintext_tag_rules('inputbox'), |
| 'tag-imagemap': plaintext_tag_rules('imagemap'), |
| 'tag-charinsert': plaintext_tag_rules('charinsert'), |
| 'tag-timeline': plaintext_tag_rules('timeline'), |
| 'tag-gallery': plaintext_tag_rules('gallery'), |
| 'tag-graph': plaintext_tag_rules('graph'), |
| 'tag-rss': plaintext_tag_rules('rss'), |
| 'tag-math': delegate_tag_rules('math', TexLexer), |
| 'tag-chem': delegate_tag_rules('chem', TexLexer), |
| 'tag-ce': delegate_tag_rules('ce', TexLexer), |
| 'tag-templatedata': delegate_tag_rules('templatedata', JsonLexer), |
| 'text-italic': text_rules(Generic.Emph), |
| 'text-bold': text_rules(Generic.Strong), |
| 'text-bold-italic': text_rules(Generic.EmphStrong), |
| 'text': text_rules(Text), |
| } |
