我们从Python开源项目中，提取了以下22个代码示例，用于说明如何使用token.NUMBER。
def test_long(self): # Long integers self.check_tokenize("x = 0", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0' (1, 4) (1, 5) """) self.check_tokenize("x = 0xfffffffffff", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0xffffffffff (1, 4) (1, 17) """) self.check_tokenize("x = 123141242151251616110", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '123141242151 (1, 4) (1, 25) """) self.check_tokenize("x = -15921590215012591", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) NUMBER '159215902150 (1, 5) (1, 22) """)
def test_additive(self): # Additive self.check_tokenize("x = 1 - y + 15 - 1 + 0x124 + z + a[5]", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '-' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '15' (1, 12) (1, 14) OP '-' (1, 15) (1, 16) NUMBER '1' (1, 17) (1, 18) OP '+' (1, 19) (1, 20) NUMBER '0x124' (1, 21) (1, 26) OP '+' (1, 27) (1, 28) NAME 'z' (1, 29) (1, 30) OP '+' (1, 31) (1, 32) NAME 'a' (1, 33) (1, 34) OP '[' (1, 34) (1, 35) NUMBER '5' (1, 35) (1, 36) OP ']' (1, 36) (1, 37) """)
def test_multiplicative(self): # Multiplicative self.check_tokenize("x = 1//1*1/5*12%0x12", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '//' (1, 5) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '/' (1, 10) (1, 11) NUMBER '5' (1, 11) (1, 12) OP '*' (1, 12) (1, 13) NUMBER '12' (1, 13) (1, 15) OP '%' (1, 15) (1, 16) NUMBER '0x12' (1, 16) (1, 20) """)
def __init__(self): self._dispatch = {} for value, name in symbol.sym_name.items(): if hasattr(self, name): self._dispatch[value] = getattr(self, name) self._dispatch[token.NEWLINE] = self.com_NEWLINE self._atom_dispatch = {token.LPAR: self.atom_lpar, token.LSQB: self.atom_lsqb, token.LBRACE: self.atom_lbrace, token.BACKQUOTE: self.atom_backquote, token.NUMBER: self.atom_number, token.STRING: self.atom_string, token.NAME: self.atom_name, } self.encoding = None
def _parse_number(self): t = self._next_token() if t.type != TokenType.NUMBER: raise ParseError("Expected number, got '%s'" % (t.value,)) if "." in t.value: return float(t.value) elif "e" in t.value.lower(): return float(t.value) elif t.value.lower().startswith("0x"): return int(t.value, 16) elif t.value.lower().startswith("0b"): return int(t.value, 2) else: return int(t.value)
def eval_token(cls, token, use_decimal=False): token_type = token.type token_text = token.string if token_type == NUMBER: try: return int(token_text) except ValueError: if use_decimal: return Decimal(token_text) return float(token_text) elif token_type == NAME: return ParserHelper.from_word(token_text) else: raise Exception('unknown token type')
def _report(ipython, msg): """Output a message prepended by a colored `Autoimport:` tag. """ # Tell prompt_toolkit to pass ANSI escapes through (PTK#187); harmless on # pre-PTK versions. sys.stdout._raw = True cs = PyColorize.Parser().color_table[ipython.colors].colors # Token.NUMBER: bright blue (cyan), looks reasonable. print("{}Autoimport:{} {}".format(cs[token.NUMBER], cs["normal"], msg))
def decistmt(s): """Substitute Decimals for floats in a string of statements. >>> from decimal import Decimal >>> s = 'print(+21.3e-5*-.1234/81.7)' >>> decistmt(s) "print (+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7'))" The format of the exponent is inherited from the platform C library. Known cases are "e-007" (Windows) and "e-07" (not Windows). Since we're only showing 11 digits, and the 12th isn't close to 5, the rest of the output should be platform-independent. >>> exec(s) #doctest: +ELLIPSIS -3.2171603427...e-0...7 Output from calculations with Decimal should be identical across all platforms. >>> exec(decistmt(s)) -3.217160342717258261933904529E-7 """ result = [] g = tokenize(BytesIO(s.encode('utf-8')).readline) # tokenize the string for toknum, tokval, _, _, _ in g: if toknum == NUMBER and '.' in tokval: # replace NUMBER tokens result.extend([ (NAME, 'Decimal'), (OP, '('), (STRING, repr(tokval)), (OP, ')') ]) else: result.append((toknum, tokval)) return untokenize(result).decode('utf-8')
def test_basic(self): self.check_tokenize("1 + 1", """\ NUMBER '1' (1, 0) (1, 1) OP '+' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) """) self.check_tokenize("if False:\n" " # NL\n" " True = False # NEWLINE\n", """\ NAME 'if' (1, 0) (1, 2) NAME 'False' (1, 3) (1, 8) OP ':' (1, 8) (1, 9) NEWLINE '\\n' (1, 9) (1, 10) COMMENT '# NL' (2, 4) (2, 8) NL '\\n' (2, 8) (2, 9) INDENT ' ' (3, 0) (3, 4) NAME 'True' (3, 4) (3, 8) OP '=' (3, 9) (3, 10) NAME 'False' (3, 11) (3, 16) COMMENT '# NEWLINE' (3, 17) (3, 26) NEWLINE '\\n' (3, 26) (3, 27) DEDENT '' (4, 0) (4, 0) """) indent_error_file = b"""\ def k(x): x += 2 x += 5 """ readline = BytesIO(indent_error_file).readline with self.assertRaisesRegex(IndentationError, "unindent does not match any " "outer indentation level"): for tok in tokenize(readline): pass
def test_float(self): # Floating point numbers self.check_tokenize("x = 3.14159", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14159' (1, 4) (1, 11) """) self.check_tokenize("x = 314159.", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '314159.' (1, 4) (1, 11) """) self.check_tokenize("x = .314159", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '.314159' (1, 4) (1, 11) """) self.check_tokenize("x = 3e14159", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3e14159' (1, 4) (1, 11) """) self.check_tokenize("x = 3E123", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3E123' (1, 4) (1, 9) """) self.check_tokenize("x+y = 3e-1230", """\ NAME 'x' (1, 0) (1, 1) OP '+' (1, 1) (1, 2) NAME 'y' (1, 2) (1, 3) OP '=' (1, 4) (1, 5) NUMBER '3e-1230' (1, 6) (1, 13) """) self.check_tokenize("x = 3.14e159", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14e159' (1, 4) (1, 12) """)
def test_function(self): self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\ NAME 'def' (1, 0) (1, 3) NAME 'd22' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ',' (1, 9) (1, 10) NAME 'b' (1, 11) (1, 12) OP ',' (1, 12) (1, 13) NAME 'c' (1, 14) (1, 15) OP '=' (1, 15) (1, 16) NUMBER '2' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) NAME 'd' (1, 19) (1, 20) OP '=' (1, 20) (1, 21) NUMBER '2' (1, 21) (1, 22) OP ',' (1, 22) (1, 23) OP '*' (1, 24) (1, 25) NAME 'k' (1, 25) (1, 26) OP ')' (1, 26) (1, 27) OP ':' (1, 27) (1, 28) NAME 'pass' (1, 29) (1, 33) """) self.check_tokenize("def d01v_(a=1, *k, **w): pass", """\ NAME 'def' (1, 0) (1, 3) NAME 'd01v_' (1, 4) (1, 9) OP '(' (1, 9) (1, 10) NAME 'a' (1, 10) (1, 11) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP ',' (1, 13) (1, 14) OP '*' (1, 15) (1, 16) NAME 'k' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) OP '**' (1, 19) (1, 21) NAME 'w' (1, 21) (1, 22) OP ')' (1, 22) (1, 23) OP ':' (1, 23) (1, 24) NAME 'pass' (1, 25) (1, 29) """)
def test_shift(self): # Shift self.check_tokenize("x = 1 << 1 >> 5", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '<<' (1, 6) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '>>' (1, 11) (1, 13) NUMBER '5' (1, 14) (1, 15) """)
def test_unary(self): # Unary self.check_tokenize("~1 ^ 1 & 1 |1 ^ -1", """\ OP '~' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '^' (1, 3) (1, 4) NUMBER '1' (1, 5) (1, 6) OP '&' (1, 7) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '|' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '^' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) NUMBER '1' (1, 17) (1, 18) """) self.check_tokenize("-1*1/1+1*1//1 - ---1**1", """\ OP '-' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '*' (1, 2) (1, 3) NUMBER '1' (1, 3) (1, 4) OP '/' (1, 4) (1, 5) NUMBER '1' (1, 5) (1, 6) OP '+' (1, 6) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '//' (1, 10) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '-' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) OP '-' (1, 17) (1, 18) OP '-' (1, 18) (1, 19) NUMBER '1' (1, 19) (1, 20) OP '**' (1, 20) (1, 22) NUMBER '1' (1, 22) (1, 23) """)
def decistmt(s): result = [] g = tokenize(BytesIO(s.encode('utf-8')).readline) # tokenize the string for toknum, tokval, _, _, _ in g: if toknum == NUMBER and '.' in tokval: # replace NUMBER tokens result.extend([ (NAME, 'Decimal'), (OP, '('), (STRING, repr(tokval)), (OP, ')') ]) else: result.append((toknum, tokval)) return untokenize(result).decode('utf-8')
def test_exact_type(self): self.assertExactTypeEqual('()', token.LPAR, token.RPAR) self.assertExactTypeEqual('[]', token.LSQB, token.RSQB) self.assertExactTypeEqual(':', token.COLON) self.assertExactTypeEqual(',', token.COMMA) self.assertExactTypeEqual(';', token.SEMI) self.assertExactTypeEqual('+', token.PLUS) self.assertExactTypeEqual('-', token.MINUS) self.assertExactTypeEqual('*', token.STAR) self.assertExactTypeEqual('/', token.SLASH) self.assertExactTypeEqual('|', token.VBAR) self.assertExactTypeEqual('&', token.AMPER) self.assertExactTypeEqual('<', token.LESS) self.assertExactTypeEqual('>', token.GREATER) self.assertExactTypeEqual('=', token.EQUAL) self.assertExactTypeEqual('.', token.DOT) self.assertExactTypeEqual('%', token.PERCENT) self.assertExactTypeEqual('{}', token.LBRACE, token.RBRACE) self.assertExactTypeEqual('==', token.EQEQUAL) self.assertExactTypeEqual('!=', token.NOTEQUAL) self.assertExactTypeEqual('<=', token.LESSEQUAL) self.assertExactTypeEqual('>=', token.GREATEREQUAL) self.assertExactTypeEqual('~', token.TILDE) self.assertExactTypeEqual('^', token.CIRCUMFLEX) self.assertExactTypeEqual('<<', token.LEFTSHIFT) self.assertExactTypeEqual('>>', token.RIGHTSHIFT) self.assertExactTypeEqual('**', token.DOUBLESTAR) self.assertExactTypeEqual('+=', token.PLUSEQUAL) self.assertExactTypeEqual('-=', token.MINEQUAL) self.assertExactTypeEqual('*=', token.STAREQUAL) self.assertExactTypeEqual('/=', token.SLASHEQUAL) self.assertExactTypeEqual('%=', token.PERCENTEQUAL) self.assertExactTypeEqual('&=', token.AMPEREQUAL) self.assertExactTypeEqual('|=', token.VBAREQUAL) self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL) self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL) self.assertExactTypeEqual('<<=', token.LEFTSHIFTEQUAL) self.assertExactTypeEqual('>>=', token.RIGHTSHIFTEQUAL) self.assertExactTypeEqual('**=', token.DOUBLESTAREQUAL) self.assertExactTypeEqual('//', token.DOUBLESLASH) self.assertExactTypeEqual('//=', token.DOUBLESLASHEQUAL) self.assertExactTypeEqual('@', token.AT) self.assertExactTypeEqual('a**2+b**2==c**2', NAME, token.DOUBLESTAR, NUMBER, token.PLUS, NAME, token.DOUBLESTAR, NUMBER, token.EQEQUAL, NAME, token.DOUBLESTAR, NUMBER) self.assertExactTypeEqual('{1, 2, 3}', token.LBRACE, token.NUMBER, token.COMMA, token.NUMBER, token.COMMA, token.NUMBER, token.RBRACE) self.assertExactTypeEqual('^(x & 0x1)', token.CIRCUMFLEX, token.LPAR, token.NAME, token.AMPER, token.NUMBER, token.RPAR)
def test_int(self): # Ordinary integers and binary operators self.check_tokenize("0xff <= 255", """\ NUMBER '0xff' (1, 0) (1, 4) OP '<=' (1, 5) (1, 7) NUMBER '255' (1, 8) (1, 11) """) self.check_tokenize("0b10 <= 255", """\ NUMBER '0b10' (1, 0) (1, 4) OP '<=' (1, 5) (1, 7) NUMBER '255' (1, 8) (1, 11) """) self.check_tokenize("0o123 <= 0O123", """\ NUMBER '0o123' (1, 0) (1, 5) OP '<=' (1, 6) (1, 8) NUMBER '0O123' (1, 9) (1, 14) """) self.check_tokenize("1234567 > ~0x15", """\ NUMBER '1234567' (1, 0) (1, 7) OP '>' (1, 8) (1, 9) OP '~' (1, 10) (1, 11) NUMBER '0x15' (1, 11) (1, 15) """) self.check_tokenize("2134568 != 1231515", """\ NUMBER '2134568' (1, 0) (1, 7) OP '!=' (1, 8) (1, 10) NUMBER '1231515' (1, 11) (1, 18) """) self.check_tokenize("(-124561-1) & 200000000", """\ OP '(' (1, 0) (1, 1) OP '-' (1, 1) (1, 2) NUMBER '124561' (1, 2) (1, 8) OP '-' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP ')' (1, 10) (1, 11) OP '&' (1, 12) (1, 13) NUMBER '200000000' (1, 14) (1, 23) """) self.check_tokenize("0xdeadbeef != -1", """\ NUMBER '0xdeadbeef' (1, 0) (1, 10) OP '!=' (1, 11) (1, 13) OP '-' (1, 14) (1, 15) NUMBER '1' (1, 15) (1, 16) """) self.check_tokenize("0xdeadc0de & 12345", """\ NUMBER '0xdeadc0de' (1, 0) (1, 10) OP '&' (1, 11) (1, 12) NUMBER '12345' (1, 13) (1, 18) """) self.check_tokenize("0xFF & 0x15 | 1234", """\ NUMBER '0xFF' (1, 0) (1, 4) OP '&' (1, 5) (1, 6) NUMBER '0x15' (1, 7) (1, 11) OP '|' (1, 12) (1, 13) NUMBER '1234' (1, 14) (1, 18) """)
