| """ |
| pygments.lexers.data |
| ~~~~~~~~~~~~~~~~~~~~ |
| |
| Lexers for data file format. |
| |
| :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS. |
| :license: BSD, see LICENSE for details. |
| """ |
| |
| from pygments.lexer import Lexer, ExtendedRegexLexer, LexerContext, \ |
| include, bygroups |
| from pygments.token import Comment, Error, Keyword, Literal, Name, Number, \ |
| Punctuation, String, Whitespace |
| |
| __all__ = ['YamlLexer', 'JsonLexer', 'JsonBareObjectLexer', 'JsonLdLexer'] |
| |
| |
| class YamlLexerContext(LexerContext): |
| """Indentation context for the YAML lexer.""" |
| |
| def __init__(self, *args, **kwds): |
| super().__init__(*args, **kwds) |
| self.indent_stack = [] |
| self.indent = -1 |
| self.next_indent = 0 |
| self.block_scalar_indent = None |
| |
| |
| class YamlLexer(ExtendedRegexLexer): |
| """ |
| Lexer for YAML, a human-friendly data serialization |
| language. |
| |
| .. versionadded:: 0.11 |
| """ |
| |
| name = 'YAML' |
| url = 'http://yaml.org/' |
| aliases = ['yaml'] |
| filenames = ['*.yaml', '*.yml'] |
| mimetypes = ['text/x-yaml'] |
| |
| def something(token_class): |
| """Do not produce empty tokens.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| if not text: |
| return |
| yield match.start(), token_class, text |
| context.pos = match.end() |
| return callback |
| |
| def reset_indent(token_class): |
| """Reset the indentation levels.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| context.indent_stack = [] |
| context.indent = -1 |
| context.next_indent = 0 |
| context.block_scalar_indent = None |
| yield match.start(), token_class, text |
| context.pos = match.end() |
| return callback |
| |
| def save_indent(token_class, start=False): |
| """Save a possible indentation level.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| extra = '' |
| if start: |
| context.next_indent = len(text) |
| if context.next_indent < context.indent: |
| while context.next_indent < context.indent: |
| context.indent = context.indent_stack.pop() |
| if context.next_indent > context.indent: |
| extra = text[context.indent:] |
| text = text[:context.indent] |
| else: |
| context.next_indent += len(text) |
| if text: |
| yield match.start(), token_class, text |
| if extra: |
| yield match.start()+len(text), token_class.Error, extra |
| context.pos = match.end() |
| return callback |
| |
| def set_indent(token_class, implicit=False): |
| """Set the previously saved indentation level.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| if context.indent < context.next_indent: |
| context.indent_stack.append(context.indent) |
| context.indent = context.next_indent |
| if not implicit: |
| context.next_indent += len(text) |
| yield match.start(), token_class, text |
| context.pos = match.end() |
| return callback |
| |
| def set_block_scalar_indent(token_class): |
| """Set an explicit indentation level for a block scalar.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| context.block_scalar_indent = None |
| if not text: |
| return |
| increment = match.group(1) |
| if increment: |
| current_indent = max(context.indent, 0) |
| increment = int(increment) |
| context.block_scalar_indent = current_indent + increment |
| if text: |
| yield match.start(), token_class, text |
| context.pos = match.end() |
| return callback |
| |
| def parse_block_scalar_empty_line(indent_token_class, content_token_class): |
| """Process an empty line in a block scalar.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| if (context.block_scalar_indent is None or |
| len(text) <= context.block_scalar_indent): |
| if text: |
| yield match.start(), indent_token_class, text |
| else: |
| indentation = text[:context.block_scalar_indent] |
| content = text[context.block_scalar_indent:] |
| yield match.start(), indent_token_class, indentation |
| yield (match.start()+context.block_scalar_indent, |
| content_token_class, content) |
| context.pos = match.end() |
| return callback |
| |
| def parse_block_scalar_indent(token_class): |
| """Process indentation spaces in a block scalar.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| if context.block_scalar_indent is None: |
| if len(text) <= max(context.indent, 0): |
| context.stack.pop() |
| context.stack.pop() |
| return |
| context.block_scalar_indent = len(text) |
| else: |
| if len(text) < context.block_scalar_indent: |
| context.stack.pop() |
| context.stack.pop() |
| return |
| if text: |
| yield match.start(), token_class, text |
| context.pos = match.end() |
| return callback |
| |
| def parse_plain_scalar_indent(token_class): |
| """Process indentation spaces in a plain scalar.""" |
| def callback(lexer, match, context): |
| text = match.group() |
| if len(text) <= context.indent: |
| context.stack.pop() |
| context.stack.pop() |
| return |
| if text: |
| yield match.start(), token_class, text |
| context.pos = match.end() |
| return callback |
| |
| tokens = { |
| # the root rules |
| 'root': [ |
| # ignored whitespaces |
| (r'[ ]+(?=#|$)', Whitespace), |
| # line breaks |
| (r'\n+', Whitespace), |
| # a comment |
| (r'#[^\n]*', Comment.Single), |
| # the '%YAML' directive |
| (r'^%YAML(?=[ ]|$)', reset_indent(Name.Tag), 'yaml-directive'), |
| # the %TAG directive |
| (r'^%TAG(?=[ ]|$)', reset_indent(Name.Tag), 'tag-directive'), |
| # document start and document end indicators |
| (r'^(?:---|\.\.\.)(?=[ ]|$)', reset_indent(Name.Namespace), |
| 'block-line'), |
| # indentation spaces |
| (r'[ ]*(?!\s|$)', save_indent(Whitespace, start=True), |
| ('block-line', 'indentation')), |
| ], |
| |
| # trailing whitespaces after directives or a block scalar indicator |
| 'ignored-line': [ |
| # ignored whitespaces |
| (r'[ ]+(?=#|$)', Whitespace), |
| # a comment |
| (r'#[^\n]*', Comment.Single), |
| # line break |
| (r'\n', Whitespace, '#pop:2'), |
| ], |
| |
| # the %YAML directive |
| 'yaml-directive': [ |
| # the version number |
| (r'([ ]+)([0-9]+\.[0-9]+)', |
| bygroups(Whitespace, Number), 'ignored-line'), |
| ], |
| |
| # the %TAG directive |
| 'tag-directive': [ |
| # a tag handle and the corresponding prefix |
| (r'([ ]+)(!|![\w-]*!)' |
| r'([ ]+)(!|!?[\w;/?:@&=+$,.!~*\'()\[\]%-]+)', |
| bygroups(Whitespace, Keyword.Type, Whitespace, Keyword.Type), |
| 'ignored-line'), |
| ], |
| |
| # block scalar indicators and indentation spaces |
| 'indentation': [ |
| # trailing whitespaces are ignored |
| (r'[ ]*$', something(Whitespace), '#pop:2'), |
| # whitespaces preceding block collection indicators |
| (r'[ ]+(?=[?:-](?:[ ]|$))', save_indent(Whitespace)), |
| # block collection indicators |
| (r'[?:-](?=[ ]|$)', set_indent(Punctuation.Indicator)), |
| # the beginning a block line |
| (r'[ ]*', save_indent(Whitespace), '#pop'), |
| ], |
| |
| # an indented line in the block context |
| 'block-line': [ |
| # the line end |
| (r'[ ]*(?=#|$)', something(Whitespace), '#pop'), |
| # whitespaces separating tokens |
| (r'[ ]+', Whitespace), |
| # key with colon |
| (r'''([^#,?\[\]{}"'\n]+)(:)(?=[ ]|$)''', |
| bygroups(Name.Tag, set_indent(Punctuation, implicit=True))), |
| # tags, anchors and aliases, |
| include('descriptors'), |
| # block collections and scalars |
| include('block-nodes'), |
| # flow collections and quoted scalars |
| include('flow-nodes'), |
| # a plain scalar |
| (r'(?=[^\s?:,\[\]{}#&*!|>\'"%@`-]|[?:-]\S)', |
| something(Name.Variable), |
| 'plain-scalar-in-block-context'), |
| ], |
| |
| # tags, anchors, aliases |
| 'descriptors': [ |
| # a full-form tag |
| (r'!<[\w#;/?:@&=+$,.!~*\'()\[\]%-]+>', Keyword.Type), |
| # a tag in the form '!', '!suffix' or '!handle!suffix' |
| (r'!(?:[\w-]+!)?' |
| r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]*', Keyword.Type), |
| # an anchor |
| (r'&[\w-]+', Name.Label), |
| # an alias |
| (r'\*[\w-]+', Name.Variable), |
| ], |
| |
| # block collections and scalars |
| 'block-nodes': [ |
| # implicit key |
| (r':(?=[ ]|$)', set_indent(Punctuation.Indicator, implicit=True)), |
| # literal and folded scalars |
| (r'[|>]', Punctuation.Indicator, |
| ('block-scalar-content', 'block-scalar-header')), |
| ], |
| |
| # flow collections and quoted scalars |
| 'flow-nodes': [ |
| # a flow sequence |
| (r'\[', Punctuation.Indicator, 'flow-sequence'), |
| # a flow mapping |
| (r'\{', Punctuation.Indicator, 'flow-mapping'), |
| # a single-quoted scalar |
| (r'\'', String, 'single-quoted-scalar'), |
| # a double-quoted scalar |
| (r'\"', String, 'double-quoted-scalar'), |
| ], |
| |
| # the content of a flow collection |
| 'flow-collection': [ |
| # whitespaces |
| (r'[ ]+', Whitespace), |
| # line breaks |
| (r'\n+', Whitespace), |
| # a comment |
| (r'#[^\n]*', Comment.Single), |
| # simple indicators |
| (r'[?:,]', Punctuation.Indicator), |
| # tags, anchors and aliases |
| include('descriptors'), |
| # nested collections and quoted scalars |
| include('flow-nodes'), |
| # a plain scalar |
| (r'(?=[^\s?:,\[\]{}#&*!|>\'"%@`])', |
| something(Name.Variable), |
| 'plain-scalar-in-flow-context'), |
| ], |
| |
| # a flow sequence indicated by '[' and ']' |
| 'flow-sequence': [ |
| # include flow collection rules |
| include('flow-collection'), |
| # the closing indicator |
| (r'\]', Punctuation.Indicator, '#pop'), |
| ], |
| |
| # a flow mapping indicated by '{' and '}' |
| 'flow-mapping': [ |
| # key with colon |
| (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''', |
| bygroups(Name.Tag, Punctuation)), |
| # include flow collection rules |
| include('flow-collection'), |
| # the closing indicator |
| (r'\}', Punctuation.Indicator, '#pop'), |
| ], |
| |
| # block scalar lines |
| 'block-scalar-content': [ |
| # line break |
| (r'\n', Whitespace), |
| # empty line |
| (r'^[ ]+$', |
| parse_block_scalar_empty_line(Whitespace, Name.Constant)), |
| # indentation spaces (we may leave the state here) |
| (r'^[ ]*', parse_block_scalar_indent(Whitespace)), |
| # line content |
| (r'[\S\t ]+', Name.Constant), |
| ], |
| |
| # the content of a literal or folded scalar |
| 'block-scalar-header': [ |
| # indentation indicator followed by chomping flag |
| (r'([1-9])?[+-]?(?=[ ]|$)', |
| set_block_scalar_indent(Punctuation.Indicator), |
| 'ignored-line'), |
| # chomping flag followed by indentation indicator |
| (r'[+-]?([1-9])?(?=[ ]|$)', |
| set_block_scalar_indent(Punctuation.Indicator), |
| 'ignored-line'), |
| ], |
| |
| # ignored and regular whitespaces in quoted scalars |
| 'quoted-scalar-whitespaces': [ |
| # leading and trailing whitespaces are ignored |
| (r'^[ ]+', Whitespace), |
| (r'[ ]+$', Whitespace), |
| # line breaks are ignored |
| (r'\n+', Whitespace), |
| # other whitespaces are a part of the value |
| (r'[ ]+', Name.Variable), |
| ], |
| |
| # single-quoted scalars |
| 'single-quoted-scalar': [ |
| # include whitespace and line break rules |
| include('quoted-scalar-whitespaces'), |
| # escaping of the quote character |
| (r'\'\'', String.Escape), |
| # regular non-whitespace characters |
| (r'[^\s\']+', String), |
| # the closing quote |
| (r'\'', String, '#pop'), |
| ], |
| |
| # double-quoted scalars |
| 'double-quoted-scalar': [ |
| # include whitespace and line break rules |
| include('quoted-scalar-whitespaces'), |
| # escaping of special characters |
| (r'\\[0abt\tn\nvfre "\\N_LP]', String), |
| # escape codes |
| (r'\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})', |
| String.Escape), |
| # regular non-whitespace characters |
| (r'[^\s"\\]+', String), |
| # the closing quote |
| (r'"', String, '#pop'), |
| ], |
| |
| # the beginning of a new line while scanning a plain scalar |
| 'plain-scalar-in-block-context-new-line': [ |
| # empty lines |
| (r'^[ ]+$', Whitespace), |
| # line breaks |
| (r'\n+', Whitespace), |
| # document start and document end indicators |
| (r'^(?=---|\.\.\.)', something(Name.Namespace), '#pop:3'), |
| # indentation spaces (we may leave the block line state here) |
| (r'^[ ]*', parse_plain_scalar_indent(Whitespace), '#pop'), |
| ], |
| |
| # a plain scalar in the block context |
| 'plain-scalar-in-block-context': [ |
| # the scalar ends with the ':' indicator |
| (r'[ ]*(?=:[ ]|:$)', something(Whitespace), '#pop'), |
| # the scalar ends with whitespaces followed by a comment |
| (r'[ ]+(?=#)', Whitespace, '#pop'), |
| # trailing whitespaces are ignored |
| (r'[ ]+$', Whitespace), |
| # line breaks are ignored |
| (r'\n+', Whitespace, 'plain-scalar-in-block-context-new-line'), |
| # other whitespaces are a part of the value |
| (r'[ ]+', Literal.Scalar.Plain), |
| # regular non-whitespace characters |
| (r'(?::(?!\s)|[^\s:])+', Literal.Scalar.Plain), |
| ], |
| |
| # a plain scalar is the flow context |
| 'plain-scalar-in-flow-context': [ |
| # the scalar ends with an indicator character |
| (r'[ ]*(?=[,:?\[\]{}])', something(Whitespace), '#pop'), |
| # the scalar ends with a comment |
| (r'[ ]+(?=#)', Whitespace, '#pop'), |
| # leading and trailing whitespaces are ignored |
| (r'^[ ]+', Whitespace), |
| (r'[ ]+$', Whitespace), |
| # line breaks are ignored |
| (r'\n+', Whitespace), |
| # other whitespaces are a part of the value |
| (r'[ ]+', Name.Variable), |
| # regular non-whitespace characters |
| (r'[^\s,:?\[\]{}]+', Name.Variable), |
| ], |
| |
| } |
| |
| def get_tokens_unprocessed(self, text=None, context=None): |
| if context is None: |
| context = YamlLexerContext(text, 0) |
| return super().get_tokens_unprocessed(text, context) |
| |
| |
| class JsonLexer(Lexer): |
| """ |
| For JSON data structures. |
| |
| Javascript-style comments are supported (like ``/* */`` and ``//``), |
| though comments are not part of the JSON specification. |
| This allows users to highlight JSON as it is used in the wild. |
| |
| No validation is performed on the input JSON document. |
| |
| .. versionadded:: 1.5 |
| """ |
| |
| name = 'JSON' |
| url = 'https://www.json.org' |
| aliases = ['json', 'json-object'] |
| filenames = ['*.json', 'Pipfile.lock'] |
| mimetypes = ['application/json', 'application/json-object'] |
| |
| # No validation of integers, floats, or constants is done. |
| # As long as the characters are members of the following |
| # sets, the token will be considered valid. For example, |
| # |
| # "--1--" is parsed as an integer |
| # "1...eee" is parsed as a float |
| # "trustful" is parsed as a constant |
| # |
| integers = set('-0123456789') |
| floats = set('.eE+') |
| constants = set('truefalsenull') # true|false|null |
| hexadecimals = set('0123456789abcdefABCDEF') |
| punctuations = set('{}[],') |
| whitespaces = {'\u0020', '\u000a', '\u000d', '\u0009'} |
| |
| def get_tokens_unprocessed(self, text): |
| """Parse JSON data.""" |
| |
| in_string = False |
| in_escape = False |
| in_unicode_escape = 0 |
| in_whitespace = False |
| in_constant = False |
| in_number = False |
| in_float = False |
| in_punctuation = False |
| in_comment_single = False |
| in_comment_multiline = False |
| expecting_second_comment_opener = False # // or /* |
| expecting_second_comment_closer = False # */ |
| |
| start = 0 |
| |
| # The queue is used to store data that may need to be tokenized |
| # differently based on what follows. In particular, JSON object |
| # keys are tokenized differently than string values, but cannot |
| # be distinguished until punctuation is encountered outside the |
| # string. |
| # |
| # A ":" character after the string indicates that the string is |
| # an object key; any other character indicates the string is a |
| # regular string value. |
| # |
| # The queue holds tuples that contain the following data: |
| # |
| # (start_index, token_type, text) |
| # |
| # By default the token type of text in double quotes is |
| # String.Double. The token type will be replaced if a colon |
| # is encountered after the string closes. |
| # |
| queue = [] |
| |
| for stop, character in enumerate(text): |
| if in_string: |
| if in_unicode_escape: |
| if character in self.hexadecimals: |
| in_unicode_escape -= 1 |
| if not in_unicode_escape: |
| in_escape = False |
| else: |
| in_unicode_escape = 0 |
| in_escape = False |
| |
| elif in_escape: |
| if character == 'u': |
| in_unicode_escape = 4 |
| else: |
| in_escape = False |
| |
| elif character == '\\': |
| in_escape = True |
| |
| elif character == '"': |
| queue.append((start, String.Double, text[start:stop + 1])) |
| in_string = False |
| in_escape = False |
| in_unicode_escape = 0 |
| |
| continue |
| |
| elif in_whitespace: |
| if character in self.whitespaces: |
| continue |
| |
| if queue: |
| queue.append((start, Whitespace, text[start:stop])) |
| else: |
| yield start, Whitespace, text[start:stop] |
| in_whitespace = False |
| # Fall through so the new character can be evaluated. |
| |
| elif in_constant: |
| if character in self.constants: |
| continue |
| |
| yield start, Keyword.Constant, text[start:stop] |
| in_constant = False |
| # Fall through so the new character can be evaluated. |
| |
| elif in_number: |
| if character in self.integers: |
| continue |
| elif character in self.floats: |
| in_float = True |
| continue |
| |
| if in_float: |
| yield start, Number.Float, text[start:stop] |
| else: |
| yield start, Number.Integer, text[start:stop] |
| in_number = False |
| in_float = False |
| # Fall through so the new character can be evaluated. |
| |
| elif in_punctuation: |
| if character in self.punctuations: |
| continue |
| |
| yield start, Punctuation, text[start:stop] |
| in_punctuation = False |
| # Fall through so the new character can be evaluated. |
| |
| elif in_comment_single: |
| if character != '\n': |
| continue |
| |
| if queue: |
| queue.append((start, Comment.Single, text[start:stop])) |
| else: |
| yield start, Comment.Single, text[start:stop] |
| |
| in_comment_single = False |
| # Fall through so the new character can be evaluated. |
| |
| elif in_comment_multiline: |
| if character == '*': |
| expecting_second_comment_closer = True |
| elif expecting_second_comment_closer: |
| expecting_second_comment_closer = False |
| if character == '/': |
| if queue: |
| queue.append((start, Comment.Multiline, text[start:stop + 1])) |
| else: |
| yield start, Comment.Multiline, text[start:stop + 1] |
| |
| in_comment_multiline = False |
| |
| continue |
| |
| elif expecting_second_comment_opener: |
| expecting_second_comment_opener = False |
| if character == '/': |
| in_comment_single = True |
| continue |
| elif character == '*': |
| in_comment_multiline = True |
| continue |
| |
| # Exhaust the queue. Accept the existing token types. |
| yield from queue |
| queue.clear() |
| |
| yield start, Error, text[start:stop] |
| # Fall through so the new character can be evaluated. |
| |
| start = stop |
| |
| if character == '"': |
| in_string = True |
| |
| elif character in self.whitespaces: |
| in_whitespace = True |
| |
| elif character in {'f', 'n', 't'}: # The first letters of true|false|null |
| # Exhaust the queue. Accept the existing token types. |
| yield from queue |
| queue.clear() |
| |
| in_constant = True |
| |
| elif character in self.integers: |
| # Exhaust the queue. Accept the existing token types. |
| yield from queue |
| queue.clear() |
| |
| in_number = True |
| |
| elif character == ':': |
| # Yield from the queue. Replace string token types. |
| for _start, _token, _text in queue: |
| # There can be only three types of tokens before a ':': |
| # Whitespace, Comment, or a quoted string. |
| # |
| # If it's a quoted string we emit Name.Tag. |
| # Otherwise, we yield the original token. |
| # |
| # In all other cases this would be invalid JSON, |
| # but this is not a validating JSON lexer, so it's OK. |
| if _token is String.Double: |
| yield _start, Name.Tag, _text |
| else: |
| yield _start, _token, _text |
| queue.clear() |
| |
| in_punctuation = True |
| |
| elif character in self.punctuations: |
| # Exhaust the queue. Accept the existing token types. |
| yield from queue |
| queue.clear() |
| |
| in_punctuation = True |
| |
| elif character == '/': |
| # This is the beginning of a comment. |
| expecting_second_comment_opener = True |
| |
| else: |
| # Exhaust the queue. Accept the existing token types. |
| yield from queue |
| queue.clear() |
| |
| yield start, Error, character |
| |
| # Yield any remaining text. |
| yield from queue |
| if in_string: |
| yield start, Error, text[start:] |
| elif in_float: |
| yield start, Number.Float, text[start:] |
| elif in_number: |
| yield start, Number.Integer, text[start:] |
| elif in_constant: |
| yield start, Keyword.Constant, text[start:] |
| elif in_whitespace: |
| yield start, Whitespace, text[start:] |
| elif in_punctuation: |
| yield start, Punctuation, text[start:] |
| elif in_comment_single: |
| yield start, Comment.Single, text[start:] |
| elif in_comment_multiline: |
| yield start, Error, text[start:] |
| elif expecting_second_comment_opener: |
| yield start, Error, text[start:] |
| |
| |
| class JsonBareObjectLexer(JsonLexer): |
| """ |
| For JSON data structures (with missing object curly braces). |
| |
| .. versionadded:: 2.2 |
| |
| .. deprecated:: 2.8.0 |
| |
| Behaves the same as `JsonLexer` now. |
| """ |
| |
| name = 'JSONBareObject' |
| aliases = [] |
| filenames = [] |
| mimetypes = [] |
| |
| |
| class JsonLdLexer(JsonLexer): |
| """ |
| For JSON-LD linked data. |
| |
| .. versionadded:: 2.0 |
| """ |
| |
| name = 'JSON-LD' |
| url = 'https://json-ld.org/' |
| aliases = ['jsonld', 'json-ld'] |
| filenames = ['*.jsonld'] |
| mimetypes = ['application/ld+json'] |
| |
| json_ld_keywords = { |
| '"@%s"' % keyword |
| for keyword in ( |
| 'base', |
| 'container', |
| 'context', |
| 'direction', |
| 'graph', |
| 'id', |
| 'import', |
| 'included', |
| 'index', |
| 'json', |
| 'language', |
| 'list', |
| 'nest', |
| 'none', |
| 'prefix', |
| 'propagate', |
| 'protected', |
| 'reverse', |
| 'set', |
| 'type', |
| 'value', |
| 'version', |
| 'vocab', |
| ) |
| } |
| |
| def get_tokens_unprocessed(self, text): |
| for start, token, value in super().get_tokens_unprocessed(text): |
| if token is Name.Tag and value in self.json_ld_keywords: |
| yield start, Name.Decorator, value |
| else: |
| yield start, token, value |
