我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用token.string()。
def exact_type(self): if self.type == OP and self.string in EXACT_TOKEN_TYPES: return EXACT_TOKEN_TYPES[self.string] else: return self.type
def tokenize(readline): """ The tokenize() generator requires one argument, readline, which must be a callable object which provides the same interface as the readline() method of built-in file objects. Each call to the function should return one line of input as bytes. Alternatively, readline can be a callable function terminating with StopIteration: readline = open(myfile, 'rb').__next__ # Example of alternate readline The generator produces 5-tuples with these members: the token type; the token string; a 2-tuple (srow, scol) of ints specifying the row and column where the token begins in the source; a 2-tuple (erow, ecol) of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed is the logical line; continuation lines are included. The first token sequence will always be an ENCODING token which tells you which encoding was used to decode the bytes stream. """ # This import is here to avoid problems when the itertools module is not # built yet and tokenize is imported. from itertools import chain, repeat encoding, consumed = detect_encoding(readline) rl_gen = iter(readline, b"") empty = repeat(b"") return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
def token_repr(tok_type, string): """Returns a human-friendly representation of a token with the given type and string.""" # repr() prefixes unicode with 'u' on Python2 but not Python3; strip it out for consistency. return '%s:%s' % (token.tok_name[tok_type], repr(string).lstrip('u'))
def __str__(self): return token_repr(self.type, self.string)
def match_token(token, tok_type, tok_str=None): """Returns true if token is of the given type and, if a string is given, has that string.""" return token.type == tok_type and (tok_str is None or token.string == tok_str)
def expect_token(token, tok_type, tok_str=None): """ Verifies that the given token is of the expected type. If tok_str is given, the token string is verified too. If the token doesn't match, raises an informative ValueError. """ if not match_token(token, tok_type, tok_str): raise ValueError("Expected token %s, got %s on line %s col %s" % ( token_repr(tok_type, tok_str), str(token), token.start[0], token.start[1] + 1))
def __repr__(self): annotated_type = '%d (%s)' % (self.type, tok_name[self.type]) return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % self._replace(type=annotated_type))
def _all_string_prefixes(): # The valid string prefixes. Only contain the lower case versions, # and don't contain any permuations (include 'fr', but not # 'rf'). The various permutations will be generated. _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] # if we add binary f-strings, add: ['fb', 'fbr'] result = set(['']) for prefix in _valid_string_prefixes: for t in _itertools.permutations(prefix): # create a list with upper and lower versions of each # character for u in _itertools.product(*[(c, c.upper()) for c in t]): result.add(''.join(u)) return result
def _compile(expr): return re.compile(expr, re.UNICODE) # Note that since _all_string_prefixes includes the empty string, # StringPrefix can be the empty string (making it optional).
def tokenize(readline): """ The tokenize() generator requires one argment, readline, which must be a callable object which provides the same interface as the readline() method of built-in file objects. Each call to the function should return one line of input as bytes. Alternately, readline can be a callable function terminating with StopIteration: readline = open(myfile, 'rb').__next__ # Example of alternate readline The generator produces 5-tuples with these members: the token type; the token string; a 2-tuple (srow, scol) of ints specifying the row and column where the token begins in the source; a 2-tuple (erow, ecol) of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed is the logical line; continuation lines are included. The first token sequence will always be an ENCODING token which tells you which encoding was used to decode the bytes stream. """ # This import is here to avoid problems when the itertools module is not # built yet and tokenize is imported. from itertools import chain, repeat encoding, consumed = detect_encoding(readline) rl_gen = iter(readline, b"") empty = repeat(b"") return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
def printDocStringData(self): """ Calls the function that parses the docstring text """ # print("Docstring text:") # for token in self.docStringText: # print(token.string, end=" ") # print("") self.parseDocStringData()
