public void testGetToken() { String data = "string: abcd"; StreamReader reader = new StreamReader(data); Scanner scanner = new ScannerImpl(reader); Mark dummy = new Mark("dummy", 0, 0, 0, "", 0); LinkedList<Token> etalonTokens = new LinkedList<Token>(); etalonTokens.add(new StreamStartToken(dummy, dummy)); etalonTokens.add(new BlockMappingStartToken(dummy, dummy)); etalonTokens.add(new KeyToken(dummy, dummy)); etalonTokens.add(new ScalarToken("string", true, dummy, dummy, (char) 0)); etalonTokens.add(new ValueToken(dummy, dummy)); etalonTokens.add(new ScalarToken("abcd", true, dummy, dummy, (char) 0)); etalonTokens.add(new BlockEndToken(dummy, dummy)); etalonTokens.add(new StreamEndToken(dummy, dummy)); while (!etalonTokens.isEmpty() && scanner.checkToken(etalonTokens.get(0).getTokenId())) { assertEquals(etalonTokens.removeFirst(), scanner.getToken()); } assertFalse("Must contain no more tokens: " + scanner.getToken(), scanner.checkToken(new Token.ID[0])); }
/** * Fetch a key in a block-style mapping. * * @see <a href="http://www.yaml.org/spec/1.1/#id863975"></a> */ private void fetchKey() { // Block context needs additional checks. if (this.flowLevel == 0) { // Are we allowed to start a key (not necessary a simple)? if (!this.allowSimpleKey) { throw new ScannerException(null, null, "mapping keys are not allowed here", reader.getMark()); } // We may need to add BLOCK-MAPPING-START. if (addIndent(this.reader.getColumn())) { Mark mark = reader.getMark(); this.tokens.add(new BlockMappingStartToken(mark, mark)); } } // Simple keys are allowed after '?' in the block context. this.allowSimpleKey = this.flowLevel == 0; // Reset possible simple key on the current level. removePossibleSimpleKey(); // Add KEY. Mark startMark = reader.getMark(); reader.forward(); Mark endMark = reader.getMark(); Token token = new KeyToken(startMark, endMark); this.tokens.add(token); }
/** * Fetch a key in a block-style mapping. * * @see http://www.yaml.org/spec/1.1/#id863975 */ private void fetchKey() { // Block context needs additional checks. if (this.flowLevel == 0) { // Are we allowed to start a key (not necessary a simple)? if (!this.allowSimpleKey) { throw new ScannerException(null, null, "mapping keys are not allowed here", reader.getMark()); } // We may need to add BLOCK-MAPPING-START. if (addIndent(this.reader.getColumn())) { Mark mark = reader.getMark(); this.tokens.add(new BlockMappingStartToken(mark, mark)); } } // Simple keys are allowed after '?' in the block context. this.allowSimpleKey = this.flowLevel == 0; // Reset possible simple key on the current level. removePossibleSimpleKey(); // Add KEY. Mark startMark = reader.getMark(); reader.forward(); Mark endMark = reader.getMark(); Token token = new KeyToken(startMark, endMark); this.tokens.add(token); }
/** * Fetch a value in a block-style mapping. * * @see <a href="http://www.yaml.org/spec/1.1/#id863975"></a> */ private void fetchValue() { // Do we determine a simple key? SimpleKey key = this.possibleSimpleKeys.remove(this.flowLevel); if (key != null) { // Add KEY. this.tokens.add(key.getTokenNumber() - this.tokensTaken, new KeyToken(key.getMark(), key.getMark())); // If this key starts a new block mapping, we need to add // BLOCK-MAPPING-START. if (this.flowLevel == 0) { if (addIndent(key.getColumn())) { this.tokens.add(key.getTokenNumber() - this.tokensTaken, new BlockMappingStartToken(key.getMark(), key.getMark())); } } // There cannot be two simple keys one after another. this.allowSimpleKey = false; } else { // It must be a part of a complex key. // Block context needs additional checks. Do we really need them? // They will be caught by the parser anyway. if (this.flowLevel == 0) { // We are allowed to start a complex value if and only if we can // start a simple key. if (!this.allowSimpleKey) { throw new ScannerException(null, null, "mapping values are not allowed here", reader.getMark()); } } // If this value starts a new block mapping, we need to add // BLOCK-MAPPING-START. It will be detected as an error later by // the parser. if (flowLevel == 0) { if (addIndent(reader.getColumn())) { Mark mark = reader.getMark(); this.tokens.add(new BlockMappingStartToken(mark, mark)); } } // Simple keys are allowed after ':' in the block context. allowSimpleKey = flowLevel == 0; // Reset possible simple key on the current level. removePossibleSimpleKey(); } // Add VALUE. Mark startMark = reader.getMark(); reader.forward(); Mark endMark = reader.getMark(); Token token = new ValueToken(startMark, endMark); this.tokens.add(token); }
/** * Fetch a value in a block-style mapping. * * @see http://www.yaml.org/spec/1.1/#id863975 */ private void fetchValue() { // Do we determine a simple key? SimpleKey key = this.possibleSimpleKeys.remove(this.flowLevel); if (key != null) { // Add KEY. this.tokens.add(key.getTokenNumber() - this.tokensTaken, new KeyToken(key.getMark(), key.getMark())); // If this key starts a new block mapping, we need to add // BLOCK-MAPPING-START. if (this.flowLevel == 0) { if (addIndent(key.getColumn())) { this.tokens.add(key.getTokenNumber() - this.tokensTaken, new BlockMappingStartToken(key.getMark(), key.getMark())); } } // There cannot be two simple keys one after another. this.allowSimpleKey = false; } else { // It must be a part of a complex key. // Block context needs additional checks. Do we really need them? // They will be caught by the parser anyway. if (this.flowLevel == 0) { // We are allowed to start a complex value if and only if we can // start a simple key. if (!this.allowSimpleKey) { throw new ScannerException(null, null, "mapping values are not allowed here", reader.getMark()); } } // If this value starts a new block mapping, we need to add // BLOCK-MAPPING-START. It will be detected as an error later by // the parser. if (flowLevel == 0) { if (addIndent(reader.getColumn())) { Mark mark = reader.getMark(); this.tokens.add(new BlockMappingStartToken(mark, mark)); } } // Simple keys are allowed after ':' in the block context. allowSimpleKey = flowLevel == 0; // Reset possible simple key on the current level. removePossibleSimpleKey(); } // Add VALUE. Mark startMark = reader.getMark(); reader.forward(); Mark endMark = reader.getMark(); Token token = new ValueToken(startMark, endMark); this.tokens.add(token); }
/** * Fetch a value in a block-style mapping. * * @see http://www.yaml.org/spec/1.1/#id863975 */ private void fetchValue() { // Do we determine a simple key? SimpleKey key = this.possibleSimpleKeys.remove(this.flowLevel); if (key != null) { // Add KEY. this.tokens.add(key.getTokenNumber() - this.tokensTaken, new KeyToken(key.getMark(), key.getMark())); // If this key starts a new block mapping, we need to add // BLOCK-MAPPING-START. if (this.flowLevel == 0) { if (addIndent(key.getColumn())) { this.tokens.add(key.getTokenNumber() - this.tokensTaken, new BlockMappingStartToken(key.getMark(), key.getMark())); } } // There cannot be two simple keys one after another. this.allowSimpleKey = false; } else {// It must be a part of a complex key. // Block context needs additional checks.Do we really need them? // They // will be catched by the parser anyway.) if (this.flowLevel == 0) { // We are allowed to start a complex value if and only if we can // start a simple key. if (!this.allowSimpleKey) { throw new ScannerException(null, null, "mapping values are not allowed here", reader.getMark()); } } // If this value starts a new block mapping, we need to add // BLOCK-MAPPING-START. It will be detected as an error later by // the parser. if (flowLevel == 0) { if (addIndent(reader.getColumn())) { Mark mark = reader.getMark(); this.tokens.add(new BlockMappingStartToken(mark, mark)); } } // Simple keys are allowed after ':' in the block context. allowSimpleKey = (flowLevel == 0); // Reset possible simple key on the current level. removePossibleSimpleKey(); } // Add VALUE. Mark startMark = reader.getMark(); reader.forward(); Mark endMark = reader.getMark(); Token token = new ValueToken(startMark, endMark); this.tokens.add(token); }
