public void testGetToken() { String data = "string: abcd"; StreamReader reader = new StreamReader(data); Scanner scanner = new ScannerImpl(reader); Mark dummy = new Mark("dummy", 0, 0, 0, "", 0); LinkedList<Token> etalonTokens = new LinkedList<Token>(); etalonTokens.add(new StreamStartToken(dummy, dummy)); etalonTokens.add(new BlockMappingStartToken(dummy, dummy)); etalonTokens.add(new KeyToken(dummy, dummy)); etalonTokens.add(new ScalarToken("string", true, dummy, dummy, (char) 0)); etalonTokens.add(new ValueToken(dummy, dummy)); etalonTokens.add(new ScalarToken("abcd", true, dummy, dummy, (char) 0)); etalonTokens.add(new BlockEndToken(dummy, dummy)); etalonTokens.add(new StreamEndToken(dummy, dummy)); while (!etalonTokens.isEmpty() && scanner.checkToken(etalonTokens.get(0).getTokenId())) { assertEquals(etalonTokens.removeFirst(), scanner.getToken()); } assertFalse("Must contain no more tokens: " + scanner.getToken(), scanner.checkToken(new Token.ID[0])); }
/** * Fetch a key in a block-style mapping. * * @see <a href="http://www.yaml.org/spec/1.1/#id863975"></a> */ private void fetchKey() { // Block context needs additional checks. if (this.flowLevel == 0) { // Are we allowed to start a key (not necessary a simple)? if (!this.allowSimpleKey) { throw new ScannerException(null, null, "mapping keys are not allowed here", reader.getMark()); } // We may need to add BLOCK-MAPPING-START. if (addIndent(this.reader.getColumn())) { Mark mark = reader.getMark(); this.tokens.add(new BlockMappingStartToken(mark, mark)); } } // Simple keys are allowed after '?' in the block context. this.allowSimpleKey = this.flowLevel == 0; // Reset possible simple key on the current level. removePossibleSimpleKey(); // Add KEY. Mark startMark = reader.getMark(); reader.forward(); Mark endMark = reader.getMark(); Token token = new KeyToken(startMark, endMark); this.tokens.add(token); }
/** * Fetch a key in a block-style mapping. * * @see http://www.yaml.org/spec/1.1/#id863975 */ private void fetchKey() { // Block context needs additional checks. if (this.flowLevel == 0) { // Are we allowed to start a key (not necessary a simple)? if (!this.allowSimpleKey) { throw new ScannerException(null, null, "mapping keys are not allowed here", reader.getMark()); } // We may need to add BLOCK-MAPPING-START. if (addIndent(this.reader.getColumn())) { Mark mark = reader.getMark(); this.tokens.add(new BlockMappingStartToken(mark, mark)); } } // Simple keys are allowed after '?' in the block context. this.allowSimpleKey = this.flowLevel == 0; // Reset possible simple key on the current level. removePossibleSimpleKey(); // Add KEY. Mark startMark = reader.getMark(); reader.forward(); Mark endMark = reader.getMark(); Token token = new KeyToken(startMark, endMark); this.tokens.add(token); }
/** * Fetch a value in a block-style mapping. * * @see <a href="http://www.yaml.org/spec/1.1/#id863975"></a> */ private void fetchValue() { // Do we determine a simple key? SimpleKey key = this.possibleSimpleKeys.remove(this.flowLevel); if (key != null) { // Add KEY. this.tokens.add(key.getTokenNumber() - this.tokensTaken, new KeyToken(key.getMark(), key.getMark())); // If this key starts a new block mapping, we need to add // BLOCK-MAPPING-START. if (this.flowLevel == 0) { if (addIndent(key.getColumn())) { this.tokens.add(key.getTokenNumber() - this.tokensTaken, new BlockMappingStartToken(key.getMark(), key.getMark())); } } // There cannot be two simple keys one after another. this.allowSimpleKey = false; } else { // It must be a part of a complex key. // Block context needs additional checks. Do we really need them? // They will be caught by the parser anyway. if (this.flowLevel == 0) { // We are allowed to start a complex value if and only if we can // start a simple key. if (!this.allowSimpleKey) { throw new ScannerException(null, null, "mapping values are not allowed here", reader.getMark()); } } // If this value starts a new block mapping, we need to add // BLOCK-MAPPING-START. It will be detected as an error later by // the parser. if (flowLevel == 0) { if (addIndent(reader.getColumn())) { Mark mark = reader.getMark(); this.tokens.add(new BlockMappingStartToken(mark, mark)); } } // Simple keys are allowed after ':' in the block context. allowSimpleKey = flowLevel == 0; // Reset possible simple key on the current level. removePossibleSimpleKey(); } // Add VALUE. Mark startMark = reader.getMark(); reader.forward(); Mark endMark = reader.getMark(); Token token = new ValueToken(startMark, endMark); this.tokens.add(token); }
private void scan() { this.tokens.add(new StreamStartToken(mark, mark)); boolean stop = false; while (!stop) { findToken(); char ch = data.charAt(index); switch (ch) { case '\0': tokens.add(new StreamEndToken(mark, mark)); stop = true; break; case '%': tokens.add(scanDirective()); break; case '-': if ("---".equals(data.substring(index, index + 3))) { index += 3; tokens.add(new DocumentStartToken(mark, mark)); } break; case '[': index++; tokens.add(new FlowSequenceStartToken(mark, mark)); break; case '{': index++; tokens.add(new FlowMappingStartToken(mark, mark)); break; case ']': index++; tokens.add(new FlowSequenceEndToken(mark, mark)); break; case '}': index++; tokens.add(new FlowMappingEndToken(mark, mark)); break; case '?': index++; tokens.add(new KeyToken(mark, mark)); break; case ':': index++; tokens.add(new ValueToken(mark, mark)); break; case ',': index++; tokens.add(new FlowEntryToken(mark, mark)); break; case '*': tokens.add(scanAlias()); break; case '&': tokens.add(scanAlias()); break; case '!': tokens.add(scanTag()); break; case '"': tokens.add(scanScalar()); break; default: throw new CanonicalException("invalid token"); } } scanned = true; }
/** * Fetch a value in a block-style mapping. * * @see http://www.yaml.org/spec/1.1/#id863975 */ private void fetchValue() { // Do we determine a simple key? SimpleKey key = this.possibleSimpleKeys.remove(this.flowLevel); if (key != null) { // Add KEY. this.tokens.add(key.getTokenNumber() - this.tokensTaken, new KeyToken(key.getMark(), key.getMark())); // If this key starts a new block mapping, we need to add // BLOCK-MAPPING-START. if (this.flowLevel == 0) { if (addIndent(key.getColumn())) { this.tokens.add(key.getTokenNumber() - this.tokensTaken, new BlockMappingStartToken(key.getMark(), key.getMark())); } } // There cannot be two simple keys one after another. this.allowSimpleKey = false; } else { // It must be a part of a complex key. // Block context needs additional checks. Do we really need them? // They will be caught by the parser anyway. if (this.flowLevel == 0) { // We are allowed to start a complex value if and only if we can // start a simple key. if (!this.allowSimpleKey) { throw new ScannerException(null, null, "mapping values are not allowed here", reader.getMark()); } } // If this value starts a new block mapping, we need to add // BLOCK-MAPPING-START. It will be detected as an error later by // the parser. if (flowLevel == 0) { if (addIndent(reader.getColumn())) { Mark mark = reader.getMark(); this.tokens.add(new BlockMappingStartToken(mark, mark)); } } // Simple keys are allowed after ':' in the block context. allowSimpleKey = flowLevel == 0; // Reset possible simple key on the current level. removePossibleSimpleKey(); } // Add VALUE. Mark startMark = reader.getMark(); reader.forward(); Mark endMark = reader.getMark(); Token token = new ValueToken(startMark, endMark); this.tokens.add(token); }
/** * Fetch a value in a block-style mapping. * * @see http://www.yaml.org/spec/1.1/#id863975 */ private void fetchValue() { // Do we determine a simple key? SimpleKey key = this.possibleSimpleKeys.remove(this.flowLevel); if (key != null) { // Add KEY. this.tokens.add(key.getTokenNumber() - this.tokensTaken, new KeyToken(key.getMark(), key.getMark())); // If this key starts a new block mapping, we need to add // BLOCK-MAPPING-START. if (this.flowLevel == 0) { if (addIndent(key.getColumn())) { this.tokens.add(key.getTokenNumber() - this.tokensTaken, new BlockMappingStartToken(key.getMark(), key.getMark())); } } // There cannot be two simple keys one after another. this.allowSimpleKey = false; } else {// It must be a part of a complex key. // Block context needs additional checks.Do we really need them? // They // will be catched by the parser anyway.) if (this.flowLevel == 0) { // We are allowed to start a complex value if and only if we can // start a simple key. if (!this.allowSimpleKey) { throw new ScannerException(null, null, "mapping values are not allowed here", reader.getMark()); } } // If this value starts a new block mapping, we need to add // BLOCK-MAPPING-START. It will be detected as an error later by // the parser. if (flowLevel == 0) { if (addIndent(reader.getColumn())) { Mark mark = reader.getMark(); this.tokens.add(new BlockMappingStartToken(mark, mark)); } } // Simple keys are allowed after ':' in the block context. allowSimpleKey = (flowLevel == 0); // Reset possible simple key on the current level. removePossibleSimpleKey(); } // Add VALUE. Mark startMark = reader.getMark(); reader.forward(); Mark endMark = reader.getMark(); Token token = new ValueToken(startMark, endMark); this.tokens.add(token); }
