| """ |
| pygments.lexers.julia |
| ~~~~~~~~~~~~~~~~~~~~~ |
| |
| Lexers for the Julia language. |
| |
| :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS. |
| :license: BSD, see LICENSE for details. |
| """ |
| |
| from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, \ |
| words, include |
| from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ |
| Number, Punctuation, Generic, Whitespace |
| from pygments.util import shebang_matches |
| from pygments.lexers._julia_builtins import OPERATORS_LIST, DOTTED_OPERATORS_LIST, \ |
| KEYWORD_LIST, BUILTIN_LIST, LITERAL_LIST |
| |
| __all__ = ['JuliaLexer', 'JuliaConsoleLexer'] |
| |
| # see https://docs.julialang.org/en/v1/manual/variables/#Allowed-Variable-Names |
| allowed_variable = \ |
| '(?:[a-zA-Z_\u00A1-\U0010ffff][a-zA-Z_0-9!\u00A1-\U0010ffff]*)' |
| # see https://github.com/JuliaLang/julia/blob/master/src/flisp/julia_opsuffs.h |
| operator_suffixes = r'[²³¹ʰʲʳʷʸˡˢˣᴬᴮᴰᴱᴳᴴᴵᴶᴷᴸᴹᴺᴼᴾᴿᵀᵁᵂᵃᵇᵈᵉᵍᵏᵐᵒᵖᵗᵘᵛᵝᵞᵟᵠᵡᵢᵣᵤᵥᵦᵧᵨᵩᵪᶜᶠᶥᶦᶫᶰᶸᶻᶿ′″‴‵‶‷⁗⁰ⁱ⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎ₐₑₒₓₕₖₗₘₙₚₛₜⱼⱽ]*' |
| |
| class JuliaLexer(RegexLexer): |
| """ |
| For Julia source code. |
| |
| .. versionadded:: 1.6 |
| """ |
| |
| name = 'Julia' |
| url = 'https://julialang.org/' |
| aliases = ['julia', 'jl'] |
| filenames = ['*.jl'] |
| mimetypes = ['text/x-julia', 'application/x-julia'] |
| |
| tokens = { |
| 'root': [ |
| (r'\n', Whitespace), |
| (r'[^\S\n]+', Whitespace), |
| (r'#=', Comment.Multiline, "blockcomment"), |
| (r'#.*$', Comment), |
| (r'[\[\](),;]', Punctuation), |
| |
| # symbols |
| # intercept range expressions first |
| (r'(' + allowed_variable + r')(\s*)(:)(' + allowed_variable + ')', |
| bygroups(Name, Whitespace, Operator, Name)), |
| # then match :name which does not follow closing brackets, digits, or the |
| # ::, <:, and :> operators |
| (r'(?<![\]):<>\d.])(:' + allowed_variable + ')', String.Symbol), |
| |
| # type assertions - excludes expressions like ::typeof(sin) and ::avec[1] |
| (r'(?<=::)(\s*)(' + allowed_variable + r')\b(?![(\[])', |
| bygroups(Whitespace, Keyword.Type)), |
| # type comparisons |
| # - MyType <: A or MyType >: A |
| ('(' + allowed_variable + r')(\s*)([<>]:)(\s*)(' + allowed_variable + r')\b(?![(\[])', |
| bygroups(Keyword.Type, Whitespace, Operator, Whitespace, Keyword.Type)), |
| # - <: B or >: B |
| (r'([<>]:)(\s*)(' + allowed_variable + r')\b(?![(\[])', |
| bygroups(Operator, Whitespace, Keyword.Type)), |
| # - A <: or A >: |
| (r'\b(' + allowed_variable + r')(\s*)([<>]:)', |
| bygroups(Keyword.Type, Whitespace, Operator)), |
| |
| # operators |
| # Suffixes aren't actually allowed on all operators, but we'll ignore that |
| # since those cases are invalid Julia code. |
| (words([*OPERATORS_LIST, *DOTTED_OPERATORS_LIST], |
| suffix=operator_suffixes), Operator), |
| (words(['.' + o for o in DOTTED_OPERATORS_LIST], |
| suffix=operator_suffixes), Operator), |
| (words(['...', '..']), Operator), |
| |
| # NOTE |
| # Patterns below work only for definition sites and thus hardly reliable. |
| # |
| # functions |
| # (r'(function)(\s+)(' + allowed_variable + ')', |
| # bygroups(Keyword, Text, Name.Function)), |
| |
| # chars |
| (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|" |
| r"\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'", String.Char), |
| |
| # try to match trailing transpose |
| (r'(?<=[.\w)\]])(\'' + operator_suffixes + ')+', Operator), |
| |
| # raw strings |
| (r'(raw)(""")', bygroups(String.Affix, String), 'tqrawstring'), |
| (r'(raw)(")', bygroups(String.Affix, String), 'rawstring'), |
| # regular expressions |
| (r'(r)(""")', bygroups(String.Affix, String.Regex), 'tqregex'), |
| (r'(r)(")', bygroups(String.Affix, String.Regex), 'regex'), |
| # other strings |
| (r'(' + allowed_variable + ')?(""")', |
| bygroups(String.Affix, String), 'tqstring'), |
| (r'(' + allowed_variable + ')?(")', |
| bygroups(String.Affix, String), 'string'), |
| |
| # backticks |
| (r'(' + allowed_variable + ')?(```)', |
| bygroups(String.Affix, String.Backtick), 'tqcommand'), |
| (r'(' + allowed_variable + ')?(`)', |
| bygroups(String.Affix, String.Backtick), 'command'), |
| |
| # type names |
| # - names that begin a curly expression |
| ('(' + allowed_variable + r')(\{)', |
| bygroups(Keyword.Type, Punctuation), 'curly'), |
| # - names as part of bare 'where' |
| (r'(where)(\s+)(' + allowed_variable + ')', |
| bygroups(Keyword, Whitespace, Keyword.Type)), |
| # - curly expressions in general |
| (r'(\{)', Punctuation, 'curly'), |
| # - names as part of type declaration |
| (r'(abstract|primitive)([ \t]+)(type\b)([\s()]+)(' + |
| allowed_variable + r')', |
| bygroups(Keyword, Whitespace, Keyword, Text, Keyword.Type)), |
| (r'(mutable(?=[ \t]))?([ \t]+)?(struct\b)([\s()]+)(' + |
| allowed_variable + r')', |
| bygroups(Keyword, Whitespace, Keyword, Text, Keyword.Type)), |
| |
| # macros |
| (r'@' + allowed_variable, Name.Decorator), |
| (words([*OPERATORS_LIST, '..', '.', *DOTTED_OPERATORS_LIST], |
| prefix='@', suffix=operator_suffixes), Name.Decorator), |
| |
| # keywords |
| (words(KEYWORD_LIST, suffix=r'\b'), Keyword), |
| # builtin types |
| (words(BUILTIN_LIST, suffix=r'\b'), Keyword.Type), |
| # builtin literals |
| (words(LITERAL_LIST, suffix=r'\b'), Name.Builtin), |
| |
| # names |
| (allowed_variable, Name), |
| |
| # numbers |
| (r'(\d+((_\d+)+)?\.(?!\.)(\d+((_\d+)+)?)?|\.\d+((_\d+)+)?)([eEf][+-]?[0-9]+)?', Number.Float), |
| (r'\d+((_\d+)+)?[eEf][+-]?[0-9]+', Number.Float), |
| (r'0x[a-fA-F0-9]+((_[a-fA-F0-9]+)+)?(\.([a-fA-F0-9]+((_[a-fA-F0-9]+)+)?)?)?p[+-]?\d+', Number.Float), |
| (r'0b[01]+((_[01]+)+)?', Number.Bin), |
| (r'0o[0-7]+((_[0-7]+)+)?', Number.Oct), |
| (r'0x[a-fA-F0-9]+((_[a-fA-F0-9]+)+)?', Number.Hex), |
| (r'\d+((_\d+)+)?', Number.Integer), |
| |
| # single dot operator matched last to permit e.g. ".1" as a float |
| (words(['.']), Operator), |
| ], |
| |
| "blockcomment": [ |
| (r'[^=#]', Comment.Multiline), |
| (r'#=', Comment.Multiline, '#push'), |
| (r'=#', Comment.Multiline, '#pop'), |
| (r'[=#]', Comment.Multiline), |
| ], |
| |
| 'curly': [ |
| (r'\{', Punctuation, '#push'), |
| (r'\}', Punctuation, '#pop'), |
| (allowed_variable, Keyword.Type), |
| include('root'), |
| ], |
| |
| 'tqrawstring': [ |
| (r'"""', String, '#pop'), |
| (r'([^"]|"[^"][^"])+', String), |
| ], |
| 'rawstring': [ |
| (r'"', String, '#pop'), |
| (r'\\"', String.Escape), |
| (r'([^"\\]|\\[^"])+', String), |
| ], |
| |
| # Interpolation is defined as "$" followed by the shortest full |
| # expression, which is something we can't parse. Include the most |
| # common cases here: $word, and $(paren'd expr). |
| 'interp': [ |
| (r'\$' + allowed_variable, String.Interpol), |
| (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'), |
| ], |
| 'in-intp': [ |
| (r'\(', Punctuation, '#push'), |
| (r'\)', Punctuation, '#pop'), |
| include('root'), |
| ], |
| |
| 'string': [ |
| (r'(")(' + allowed_variable + r'|\d+)?', |
| bygroups(String, String.Affix), '#pop'), |
| # FIXME: This escape pattern is not perfect. |
| (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape), |
| include('interp'), |
| # @printf and @sprintf formats |
| (r'%[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]', |
| String.Interpol), |
| (r'[^"$%\\]+', String), |
| (r'.', String), |
| ], |
| 'tqstring': [ |
| (r'(""")(' + allowed_variable + r'|\d+)?', |
| bygroups(String, String.Affix), '#pop'), |
| (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape), |
| include('interp'), |
| (r'[^"$%\\]+', String), |
| (r'.', String), |
| ], |
| |
| 'regex': [ |
| (r'(")([imsxa]*)?', bygroups(String.Regex, String.Affix), '#pop'), |
| (r'\\"', String.Regex), |
| (r'[^\\"]+', String.Regex), |
| ], |
| |
| 'tqregex': [ |
| (r'(""")([imsxa]*)?', bygroups(String.Regex, String.Affix), '#pop'), |
| (r'[^"]+', String.Regex), |
| ], |
| |
| 'command': [ |
| (r'(`)(' + allowed_variable + r'|\d+)?', |
| bygroups(String.Backtick, String.Affix), '#pop'), |
| (r'\\[`$]', String.Escape), |
| include('interp'), |
| (r'[^\\`$]+', String.Backtick), |
| (r'.', String.Backtick), |
| ], |
| 'tqcommand': [ |
| (r'(```)(' + allowed_variable + r'|\d+)?', |
| bygroups(String.Backtick, String.Affix), '#pop'), |
| (r'\\\$', String.Escape), |
| include('interp'), |
| (r'[^\\`$]+', String.Backtick), |
| (r'.', String.Backtick), |
| ], |
| } |
| |
| def analyse_text(text): |
| return shebang_matches(text, r'julia') |
| |
| |
| class JuliaConsoleLexer(Lexer): |
| """ |
| For Julia console sessions. Modeled after MatlabSessionLexer. |
| |
| .. versionadded:: 1.6 |
| """ |
| name = 'Julia console' |
| aliases = ['jlcon', 'julia-repl'] |
| |
| def get_tokens_unprocessed(self, text): |
| jllexer = JuliaLexer(**self.options) |
| start = 0 |
| curcode = '' |
| insertions = [] |
| output = False |
| error = False |
| |
| for line in text.splitlines(keepends=True): |
| if line.startswith('julia>'): |
| insertions.append((len(curcode), [(0, Generic.Prompt, line[:6])])) |
| curcode += line[6:] |
| output = False |
| error = False |
| elif line.startswith('help?>') or line.startswith('shell>'): |
| yield start, Generic.Prompt, line[:6] |
| yield start + 6, Text, line[6:] |
| output = False |
| error = False |
| elif line.startswith(' ') and not output: |
| insertions.append((len(curcode), [(0, Whitespace, line[:6])])) |
| curcode += line[6:] |
| else: |
| if curcode: |
| yield from do_insertions( |
| insertions, jllexer.get_tokens_unprocessed(curcode)) |
| curcode = '' |
| insertions = [] |
| if line.startswith('ERROR: ') or error: |
| yield start, Generic.Error, line |
| error = True |
| else: |
| yield start, Generic.Output, line |
| output = True |
| start += len(line) |
| |
| if curcode: |
| yield from do_insertions( |
| insertions, jllexer.get_tokens_unprocessed(curcode)) |
