Demo entry 2375077

Some pygments code


Submitted by anonymous on Aug 16, 2015 at 13:59
Language: Python. Code size: 2.7 kB.

# -*- coding: utf-8 -*-

    Basic token types and the standard tokens.

    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.

class _TokenType(tuple):
    parent = None

    def split(self):
        buf = []
        node = self
        while node is not None:
            node = node.parent
        return buf

    def __init__(self, *args):
        # no need to call super.__init__
        self.subtypes = set()

    def __contains__(self, val):
        return self is val or (
            type(val) is self.__class__ and
            val[:len(self)] == self

    def __getattr__(self, val):
        if not val or not val[0].isupper():
            return tuple.__getattribute__(self, val)
        new = _TokenType(self + (val,))
        setattr(self, val, new)
        new.parent = self
        return new

    def __repr__(self):
        return 'Token' + (self and '.' or '') + '.'.join(self)

Token       = _TokenType()

# Special token types
Text        = Token.Text
Whitespace  = Text.Whitespace
Escape      = Token.Escape
Error       = Token.Error
# Text that doesn't belong to this lexer (e.g. HTML in PHP)
Other       = Token.Other

# Common token types for source code
Keyword     = Token.Keyword
Name        = Token.Name
Literal     = Token.Literal
String      = Literal.String
Number      = Literal.Number
Punctuation = Token.Punctuation
Operator    = Token.Operator
Comment     = Token.Comment

# Generic types for non-source code
Generic     = Token.Generic

# String and some others are not direct childs of Token.
# alias them:
Token.Token = Token
Token.String = String
Token.Number = Number

def is_token_subtype(ttype, other):
    Return True if ``ttype`` is a subtype of ``other``.

    exists for backwards compatibility. use ``ttype in other`` now.
    return ttype in other

def string_to_tokentype(s):
    Convert a string into a token type::

        >>> string_to_token('String.Double')
        >>> string_to_token('Token.Literal.Number')
        >>> string_to_token('')

    Tokens that are already tokens are returned unchanged:

        >>> string_to_token(String)
    if isinstance(s, _TokenType):
        return s
    if not s:
        return Token
    node = Token
    for item in s.split('.'):
        node = getattr(node, item)
    return node

This snippet took 0.01 seconds to highlight.

Back to the Entry List or Home.

Delete this entry (admin only).