我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用token.NAME。
def _getname(g): # Helper to get a dotted name, return a pair (name, token) where # name is the dotted name, or None if there was no dotted name, # and token is the next input token. parts = [] tokentype, token = g.next()[0:2] if tokentype != NAME and token != '*': return (None, token) parts.append(token) while True: tokentype, token = g.next()[0:2] if token != '.': break tokentype, token = g.next()[0:2] if tokentype != NAME: break parts.append(token) return (".".join(parts), token)
def comparison(cls, nodelist): if len(nodelist)>4: raise SyntaxError("Chained comparison not allowed in environment markers") comp = nodelist[2][1] cop = comp[1] if comp[0] == token.NAME: if len(nodelist[2]) == 3: if cop == 'not': cop = 'not in' else: cop = 'is not' try: cop = cls.get_op(cop) except KeyError: raise SyntaxError(repr(cop)+" operator not allowed in environment markers") return cop(cls.evaluate(nodelist[1]), cls.evaluate(nodelist[3]))
def evaluate(cls, nodelist): while len(nodelist)==2: nodelist = nodelist[1] kind = nodelist[0] name = nodelist[1] if kind==token.NAME: try: op = cls.values[name] except KeyError: raise SyntaxError("Unknown name %r" % name) return op() if kind==token.STRING: s = nodelist[1] if s[:1] not in "'\"" or s.startswith('"""') or s.startswith("'''") \ or '\\' in s: raise SyntaxError( "Only plain strings allowed in environment markers") return s[1:-1] raise SyntaxError("Language feature not supported in environment markers")
def comparison(cls, nodelist): if len(nodelist) > 4: msg = "Chained comparison not allowed in environment markers" raise SyntaxError(msg) comp = nodelist[2][1] cop = comp[1] if comp[0] == token.NAME: if len(nodelist[2]) == 3: if cop == 'not': cop = 'not in' else: cop = 'is not' try: cop = cls.get_op(cop) except KeyError: msg = repr(cop) + " operator not allowed in environment markers" raise SyntaxError(msg) return cop(cls.evaluate(nodelist[1]), cls.evaluate(nodelist[3]))
def evaluate(cls, nodelist): while len(nodelist)==2: nodelist = nodelist[1] kind = nodelist[0] name = nodelist[1] if kind==token.NAME: try: op = cls.values[name] except KeyError: raise SyntaxError("Unknown name %r" % name) return op() if kind==token.STRING: s = nodelist[1] if not cls._safe_string(s): raise SyntaxError( "Only plain strings allowed in environment markers") return s[1:-1] msg = "Language feature not supported in environment markers" raise SyntaxError(msg)
def classdef(self, nodelist): # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite name = nodelist[1][1] doc = self.get_docstring(nodelist[-1]) if nodelist[2][0] == token.COLON: bases = [] elif nodelist[3][0] == token.RPAR: bases = [] else: bases = self.com_bases(nodelist[3]) # code for class code = self.com_node(nodelist[-1]) if doc is not None: assert isinstance(code, Stmt) assert isinstance(code.nodes[0], Discard) del code.nodes[0] return Class(name, bases, doc, code, lineno=nodelist[1][2])
def com_argument(self, nodelist, kw, star_node): if len(nodelist) == 3 and nodelist[2][0] == symbol.comp_for: test = self.com_node(nodelist[1]) return 0, self.com_generator_expression(test, nodelist[2]) if len(nodelist) == 2: if kw: raise SyntaxError, "non-keyword arg after keyword arg" if star_node: raise SyntaxError, "only named arguments may follow *expression" return 0, self.com_node(nodelist[1]) result = self.com_node(nodelist[3]) n = nodelist[1] while len(n) == 2 and n[0] != token.NAME: n = n[1] if n[0] != token.NAME: raise SyntaxError, "keyword can't be an expression (%s)"%n[0] node = Keyword(n[1], result, lineno=n[2]) return 1, node
def test_tokenizing(self): # Test that we produce meaningful tokens on initialization. source = "import re # comment\n\nfoo = 'bar'\n" atok = asttokens.ASTTokens(source) self.assertEqual(atok.text, source) self.assertEqual([str(t) for t in atok.tokens], [ "NAME:'import'", "NAME:'re'", "COMMENT:'# comment'", "NEWLINE:'\\n'", "NL:'\\n'", "NAME:'foo'", "OP:'='", 'STRING:"\'bar\'"', "NEWLINE:'\\n'", "ENDMARKER:''" ]) self.assertEqual(atok.tokens[5].type, token.NAME) self.assertEqual(atok.tokens[5].string, 'foo') self.assertEqual(atok.tokens[5].index, 5) self.assertEqual(atok.tokens[5].startpos, 22) self.assertEqual(atok.tokens[5].endpos, 25)
def _getname(g): # Helper to get a dotted name, return a pair (name, token) where # name is the dotted name, or None if there was no dotted name, # and token is the next input token. parts = [] tokentype, token = next(g)[0:2] if tokentype != NAME and token != '*': return (None, token) parts.append(token) while True: tokentype, token = next(g)[0:2] if token != '.': break tokentype, token = next(g)[0:2] if tokentype != NAME: break parts.append(token) return (".".join(parts), token)
def process_parentheses(token, previous_token): if token.string == '(': is_function = ( previous_token and ( (previous_token.string in CLOSE_ATOM_STRINGS) or ( previous_token.type == mod_token.NAME and previous_token.string not in ALL_KWDS ) ) ) if is_function: tk_string = previous_token.string if tk_string in NOT_PYTHON_2_KWDS: return [PY2_ONLY_ERROR] if tk_string in NOT_PYTHON_3_KWDS: return [PY3K_ONLY_ERROR] else: return [TUPLE_OR_PARENTH_FORM] return [True]
