/** * Scan a flow-style scalar. Flow scalars are presented in one of two forms; * first, a flow scalar may be a double-quoted string; second, a flow scalar * may be a single-quoted string. * * @see <a href="http://www.yaml.org/spec/1.1/#flow"></a> style/syntax * * <pre> * See the specification for details. * Note that we loose indentation rules for quoted scalars. Quoted * scalars don't need to adhere indentation because " and ' clearly * mark the beginning and the end of them. Therefore we are less * restrictive then the specification requires. We only need to check * that document separators are not included in scalars. * </pre> */ private Token scanFlowScalar(char style) { boolean _double; // The style will be either single- or double-quoted; we determine this // by the first character in the entry (supplied) if (style == '"') { _double = true; } else { _double = false; } StringBuilder chunks = new StringBuilder(); Mark startMark = reader.getMark(); char quote = reader.peek(); reader.forward(); chunks.append(scanFlowScalarNonSpaces(_double, startMark)); while (reader.peek() != quote) { chunks.append(scanFlowScalarSpaces(startMark)); chunks.append(scanFlowScalarNonSpaces(_double, startMark)); } reader.forward(); Mark endMark = reader.getMark(); return new ScalarToken(chunks.toString(), false, startMark, endMark, style); }
/** * Scan a flow-style scalar. Flow scalars are presented in one of two forms; * first, a flow scalar may be a double-quoted string; second, a flow scalar * may be a single-quoted string. * * @see <a href="http://www.yaml.org/spec/1.1/#flow"></a> style/syntax * * <pre> * See the specification for details. * Note that we loose indentation rules for quoted scalars. Quoted * scalars don't need to adhere indentation because " and ' clearly * mark the beginning and the end of them. Therefore we are less * restrictive then the specification requires. We only need to check * that document separators are not included in scalars. * </pre> */ private Token scanFlowScalar(char style) { boolean _double; // The style will be either single- or double-quoted; we determine this // by the first character in the entry (supplied) if (style == '"') { _double = true; } else { _double = false; } StringBuilder chunks = new StringBuilder(); Mark startMark = reader.getMark(); int quote = reader.peek(); reader.forward(); chunks.append(scanFlowScalarNonSpaces(_double, startMark)); while (reader.peek() != quote) { chunks.append(scanFlowScalarSpaces(startMark)); chunks.append(scanFlowScalarNonSpaces(_double, startMark)); } reader.forward(); Mark endMark = reader.getMark(); return new ScalarToken(chunks.toString(), false, startMark, endMark, style); }
public void testGetToken() { String data = "string: abcd"; StreamReader reader = new StreamReader(data); Scanner scanner = new ScannerImpl(reader); Mark dummy = new Mark("dummy", 0, 0, 0, "", 0); LinkedList<Token> etalonTokens = new LinkedList<Token>(); etalonTokens.add(new StreamStartToken(dummy, dummy)); etalonTokens.add(new BlockMappingStartToken(dummy, dummy)); etalonTokens.add(new KeyToken(dummy, dummy)); etalonTokens.add(new ScalarToken("string", true, dummy, dummy, (char) 0)); etalonTokens.add(new ValueToken(dummy, dummy)); etalonTokens.add(new ScalarToken("abcd", true, dummy, dummy, (char) 0)); etalonTokens.add(new BlockEndToken(dummy, dummy)); etalonTokens.add(new StreamEndToken(dummy, dummy)); while (!etalonTokens.isEmpty() && scanner.checkToken(etalonTokens.get(0).getTokenId())) { assertEquals(etalonTokens.removeFirst(), scanner.getToken()); } assertFalse("Must contain no more tokens: " + scanner.getToken(), scanner.checkToken(new Token.ID[0])); }
/** * Scan a flow-style scalar. Flow scalars are presented in one of two forms; * first, a flow scalar may be a double-quoted string; second, a flow scalar * may be a single-quoted string. * * @see http://www.yaml.org/spec/1.1/#flow style/syntax * * <pre> * See the specification for details. * Note that we loose indentation rules for quoted scalars. Quoted * scalars don't need to adhere indentation because " and ' clearly * mark the beginning and the end of them. Therefore we are less * restrictive then the specification requires. We only need to check * that document separators are not included in scalars. * </pre> */ private Token scanFlowScalar(char style) { boolean _double; // The style will be either single- or double-quoted; we determine this // by the first character in the entry (supplied) if (style == '"') { _double = true; } else { _double = false; } StringBuilder chunks = new StringBuilder(); Mark startMark = reader.getMark(); char quote = reader.peek(); reader.forward(); chunks.append(scanFlowScalarNonSpaces(_double, startMark)); while (reader.peek() != quote) { chunks.append(scanFlowScalarSpaces(startMark)); chunks.append(scanFlowScalarNonSpaces(_double, startMark)); } reader.forward(); Mark endMark = reader.getMark(); return new ScalarToken(chunks.toString(), false, startMark, endMark, style); }
/** * Scan a flow-style scalar. Flow scalars are presented in one of two forms; first, a flow scalar may be a double-quoted string; second, * a flow scalar may be a single-quoted string. * * @see http://www.yaml.org/spec/1.1/#flow style/syntax * * <pre> * See the specification for details. * Note that we loose indentation rules for quoted scalars. Quoted * scalars don't need to adhere indentation because " and ' clearly * mark the beginning and the end of them. Therefore we are less * restrictive then the specification requires. We only need to check * that document separators are not included in scalars. * </pre> */ private Token scanFlowScalar(final char style) { boolean _double; // The style will be either single- or double-quoted; we determine this // by the first character in the entry (supplied) if (style == '"') { _double = true; } else { _double = false; } StringBuilder chunks = new StringBuilder(); Mark startMark = reader.getMark(); char quote = reader.peek(); reader.forward(); chunks.append(scanFlowScalarNonSpaces(_double, startMark)); while (reader.peek() != quote) { chunks.append(scanFlowScalarSpaces(startMark)); chunks.append(scanFlowScalarNonSpaces(_double, startMark)); } reader.forward(); Mark endMark = reader.getMark(); return new ScalarToken(chunks.toString(), false, startMark, endMark, style); }
/** * Scan a plain scalar. * * <pre> * See the specification for details. * We add an additional restriction for the flow context: * plain scalars in the flow context cannot contain ',', ':' and '?'. * We also keep track of the `allow_simple_key` flag here. * Indentation rules are loosed for the flow context. * </pre> */ private Token scanPlain() { StringBuilder chunks = new StringBuilder(); Mark startMark = reader.getMark(); Mark endMark = startMark; int indent = this.indent + 1; String spaces = ""; while (true) { char ch; int length = 0; // A comment indicates the end of the scalar. if (reader.peek() == '#') { break; } while (true) { ch = reader.peek(length); if (Constant.NULL_BL_T_LINEBR.has(ch) || (this.flowLevel == 0 && ch == ':' && Constant.NULL_BL_T_LINEBR .has(reader.peek(length + 1))) || (this.flowLevel != 0 && ",:?[]{}".indexOf(ch) != -1)) { break; } length++; } // It's not clear what we should do with ':' in the flow context. if (this.flowLevel != 0 && ch == ':' && Constant.NULL_BL_T_LINEBR.hasNo(reader.peek(length + 1), ",[]{}")) { reader.forward(length); throw new ScannerException("while scanning a plain scalar", startMark, "found unexpected ':'", reader.getMark(), "Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details."); } if (length == 0) { break; } this.allowSimpleKey = false; chunks.append(spaces); chunks.append(reader.prefixForward(length)); endMark = reader.getMark(); spaces = scanPlainSpaces(); // System.out.printf("spaces[%s]\n", spaces); if (spaces.length() == 0 || reader.peek() == '#' || (this.flowLevel == 0 && this.reader.getColumn() < indent)) { break; } } return new ScalarToken(chunks.toString(), startMark, endMark, true); }
/** * Scan a plain scalar. * * <pre> * See the specification for details. * We add an additional restriction for the flow context: * plain scalars in the flow context cannot contain ',', ':' and '?'. * We also keep track of the `allow_simple_key` flag here. * Indentation rules are loosed for the flow context. * </pre> */ private Token scanPlain() { StringBuilder chunks = new StringBuilder(); Mark startMark = reader.getMark(); Mark endMark = startMark; int indent = this.indent + 1; String spaces = ""; while (true) { int c; int length = 0; // A comment indicates the end of the scalar. if (reader.peek() == '#') { break; } while (true) { c = reader.peek(length); if (Constant.NULL_BL_T_LINEBR.has(c) || (this.flowLevel == 0 && c == ':' && Constant.NULL_BL_T_LINEBR .has(reader.peek(length + 1))) || (this.flowLevel != 0 && ",:?[]{}".indexOf(c) != -1)) { break; } length++; } // It's not clear what we should do with ':' in the flow context. if (this.flowLevel != 0 && c == ':' && Constant.NULL_BL_T_LINEBR.hasNo(reader.peek(length + 1), ",[]{}")) { reader.forward(length); throw new ScannerException("while scanning a plain scalar", startMark, "found unexpected ':'", reader.getMark(), "Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details."); } if (length == 0) { break; } this.allowSimpleKey = false; chunks.append(spaces); chunks.append(reader.prefixForward(length)); endMark = reader.getMark(); spaces = scanPlainSpaces(); // System.out.printf("spaces[%s]\n", spaces); if (spaces.length() == 0 || reader.peek() == '#' || (this.flowLevel == 0 && this.reader.getColumn() < indent)) { break; } } return new ScalarToken(chunks.toString(), startMark, endMark, true); }
private Token scanScalar() { index++; StringBuilder chunks = new StringBuilder(); int start = index; boolean ignoreSpaces = false; while (data.charAt(index) != '"') { if (data.charAt(index) == '\\') { ignoreSpaces = false; chunks.append(data.substring(start, index)); index++; char ch = data.charAt(index); index++; if (ch == '\n') { ignoreSpaces = true; } else if (QUOTE_CODES.keySet().contains(ch)) { int length = QUOTE_CODES.get(ch); int code = Integer.parseInt(data.substring(index, index + length), 16); chunks.append(String.valueOf((char) code)); index += length; } else { if (!QUOTE_REPLACES.keySet().contains(ch)) { throw new CanonicalException("invalid escape code"); } chunks.append(QUOTE_REPLACES.get(ch)); } start = index; } else if (data.charAt(index) == '\n') { chunks.append(data.substring(start, index)); chunks.append(" "); index++; start = index; ignoreSpaces = true; } else if (ignoreSpaces && data.charAt(index) == ' ') { index++; start = index; } else { ignoreSpaces = false; index++; } } chunks.append(data.substring(start, index)); index++; return new ScalarToken(chunks.toString(), mark, mark, false); }
/** * Scan a plain scalar. * * <pre> * See the specification for details. * We add an additional restriction for the flow context: * plain scalars in the flow context cannot contain ',', ':' and '?'. * We also keep track of the `allow_simple_key` flag here. * Indentation rules are loosed for the flow context. * </pre> */ private Token scanPlain() { StringBuilder chunks = new StringBuilder(); Mark startMark = reader.getMark(); Mark endMark = startMark; int indent = this.indent + 1; String spaces = ""; while (true) { char ch; int length = 0; // A comment indicates the end of the scalar. if (reader.peek() == '#') { break; } while (true) { ch = reader.peek(length); if (Constant.NULL_BL_T_LINEBR.has(ch) || (this.flowLevel == 0 && ch == ':' && Constant.NULL_BL_T_LINEBR.has(reader.peek(length + 1))) || (this.flowLevel != 0 && ",:?[]{}".indexOf(ch) != -1)) { break; } length++; } // It's not clear what we should do with ':' in the flow context. if (this.flowLevel != 0 && ch == ':' && Constant.NULL_BL_T_LINEBR.hasNo(reader.peek(length + 1), ",[]{}")) { reader.forward(length); throw new ScannerException("while scanning a plain scalar", startMark, "found unexpected ':'", reader.getMark(), "Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details."); } if (length == 0) { break; } this.allowSimpleKey = false; chunks.append(spaces); chunks.append(reader.prefixForward(length)); endMark = reader.getMark(); spaces = scanPlainSpaces(); // System.out.printf("spaces[%s]\n", spaces); if (spaces.length() == 0 || reader.peek() == '#' || (this.flowLevel == 0 && this.reader.getColumn() < indent)) { break; } } return new ScalarToken(chunks.toString(), startMark, endMark, true); }