我们从Python开源项目中，提取了以下10个代码示例，用于说明如何使用token.ENDMARKER。
def restart(self, string): """ Restarts parsing with new string Returns self for chaining. """ try: self.tokens = [ ActionParser.Token(type, string) for (type, string, trash, trash, trash) in generate_tokens( iter([string]).next ) if type != TokenType.ENDMARKER ] except TokenError: self.tokens = None self.index = 0 return self
def file_input(self, nodelist): doc = self.get_docstring(nodelist, symbol.file_input) if doc is not None: i = 1 else: i = 0 stmts = [] for node in nodelist[i:]: if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: self.com_append_stmt(stmts, node) return Module(doc, Stmt(stmts))
def _find_last_in_line(self, start_token): try: newline = self._code.find_token(start_token, token.NEWLINE) except IndexError: newline = self._code.find_token(start_token, token.ENDMARKER) return self._code.prev_token(newline)
def _simulate_compile_singlemode(self, s): # Calculate line offsets lines = [0, 0] pos = 0 while 1: pos = s.find('\n', pos)+1 if not pos: break lines.append(pos) lines.append(len(s)) oldpos = 0 parenlevel = 0 deflevel = 0 output = [] stmt = [] text = StringIO(s) tok_gen = tokenize.generate_tokens(text.readline) for toktype, tok, (srow,scol), (erow,ecol), line in tok_gen: newpos = lines[srow] + scol stmt.append(s[oldpos:newpos]) if tok != '': stmt.append(tok) oldpos = newpos + len(tok) # Update the paren level. if tok in '([{': parenlevel += 1 if tok in '}])': parenlevel -= 1 if tok in ('def', 'class') and deflevel == 0: deflevel = 1 if deflevel and toktype == token.INDENT: deflevel += 1 if deflevel and toktype == token.DEDENT: deflevel -= 1 # Are we starting a statement? if ((toktype in (token.NEWLINE, tokenize.NL, tokenize.COMMENT, token.INDENT, token.ENDMARKER) or tok==':') and parenlevel == 0): if deflevel == 0 and self._is_expr(stmt[1:-2]): output += stmt[0] output.append('__print__((') output += stmt[1:-2] output.append('))') output += stmt[-2:] else: output += stmt stmt = [] return ''.join(output)
