@Override public void reflectWith(AttributeReflector reflector) { super.reflectWith(reflector); reflector.reflect(OffsetAttribute.class, "startOffset", startOffset); reflector.reflect(OffsetAttribute.class, "endOffset", endOffset); reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", positionIncrement); reflector.reflect(PositionLengthAttribute.class, "positionLength", positionLength); reflector.reflect(TypeAttribute.class, "type", type); }
@Override public void reflectWith(AttributeReflector reflector) { fillBytesRef(); reflector.reflect(TermToBytesRefAttribute.class, "bytes", bytes.toBytesRef()); reflector.reflect(NumericTermAttribute.class, "shift", shift); reflector.reflect(NumericTermAttribute.class, "rawValue", getRawValue()); reflector.reflect(NumericTermAttribute.class, "valueSize", valueSize); }
/** * other attribute extract object. * Extracted object group by AttributeClassName * * @param stream current TokenStream * @param includeAttributes filtering attributes * @return Map<key value> */ private static Map<String, Object> extractExtendedAttributes(TokenStream stream, final Set<String> includeAttributes) { final Map<String, Object> extendedAttributes = new TreeMap<>(); stream.reflectWith(new AttributeReflector() { @Override public void reflect(Class<? extends Attribute> attClass, String key, Object value) { if (CharTermAttribute.class.isAssignableFrom(attClass)) return; if (PositionIncrementAttribute.class.isAssignableFrom(attClass)) return; if (OffsetAttribute.class.isAssignableFrom(attClass)) return; if (TypeAttribute.class.isAssignableFrom(attClass)) return; if (includeAttributes == null || includeAttributes.isEmpty() || includeAttributes.contains(key.toLowerCase(Locale.ROOT))) { if (value instanceof BytesRef) { final BytesRef p = (BytesRef) value; value = p.toString(); } extendedAttributes.put(key, value); } } }); return extendedAttributes; }
@Override public void reflectWith(AttributeReflector reflector) { super.reflectWith(reflector); reflector.reflect(OffsetAttribute.class, "startOffset", startOffset); reflector.reflect(OffsetAttribute.class, "endOffset", endOffset); reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", positionIncrement); reflector.reflect(PayloadAttribute.class, "payload", payload); reflector.reflect(FlagsAttribute.class, "flags", flags); reflector.reflect(TypeAttribute.class, "type", type); }
@Override public void reflectWith(AttributeReflector reflector) { reflector.reflect(StemmingBufferAttribute.class, "stemmedToken", stemmedToken); reflector.reflect(StemmingBufferAttribute.class, "stemmedTokenLength", stemmedTokenLength); reflector.reflect(StemmingBufferAttribute.class, "originalToken", originalToken); reflector.reflect(StemmingBufferAttribute.class, "originalTokenLength", originalTokenLength); }
@Override public void reflectWith(AttributeReflector reflector) { String reading = getReading(); String readingEN = reading == null ? null : ToStringUtil.getRomanization(reading); String pronunciation = getPronunciation(); String pronunciationEN = pronunciation == null ? null : ToStringUtil.getRomanization(pronunciation); reflector.reflect(ReadingAttribute.class, "reading", reading); reflector.reflect(ReadingAttribute.class, "reading (en)", readingEN); reflector.reflect(ReadingAttribute.class, "pronunciation", pronunciation); reflector.reflect(ReadingAttribute.class, "pronunciation (en)", pronunciationEN); }
@Override public void reflectWith(AttributeReflector reflector) { String partOfSpeech = getPartOfSpeech(); String partOfSpeechEN = partOfSpeech == null ? null : ToStringUtil.getPOSTranslation(partOfSpeech); reflector.reflect(PartOfSpeechAttribute.class, "partOfSpeech", partOfSpeech); reflector.reflect(PartOfSpeechAttribute.class, "partOfSpeech (en)", partOfSpeechEN); }
@Override public void reflectWith(AttributeReflector reflector) { String type = getInflectionType(); String typeEN = type == null ? null : ToStringUtil.getInflectionTypeTranslation(type); reflector.reflect(InflectionAttribute.class, "inflectionType", type); reflector.reflect(InflectionAttribute.class, "inflectionType (en)", typeEN); String form = getInflectionForm(); String formEN = form == null ? null : ToStringUtil.getInflectedFormTranslation(form); reflector.reflect(InflectionAttribute.class, "inflectionForm", form); reflector.reflect(InflectionAttribute.class, "inflectionForm (en)", formEN); }
@Override public void reflectWith(AttributeReflector reflector) { // when wordbreaking CJK, we use the 15924 code Japanese (Han+Hiragana+Katakana) to // mark runs of Chinese/Japanese. our use is correct (as for chinese Han is a subset), // but this is just to help prevent confusion. String name = code == UScript.JAPANESE ? "Chinese/Japanese" : getName(); reflector.reflect(ScriptAttribute.class, "script", name); }
@Override public void reflectWith(AttributeReflector reflector) { List<String> readings = getReadings(); List<String> enReadings = null; if (readings != null) { enReadings = new ArrayList<String>(readings.size()); for (String kana : readings) { enReadings.add(ToStringUtil.getRomanization(kana)); } } reflector.reflect(ReadingsAttribute.class, "readings", readings); reflector.reflect(ReadingsAttribute.class, "readings (en)", enReadings); }
@Override public void reflectWith(AttributeReflector reflector) { String partOfSpeech = getPartOfSpeech(); String en = partOfSpeech == null ? null : ToStringUtil.getPOSTranslation(partOfSpeech); reflector.reflect(PartOfSpeechAttribute.class, "partOfSpeech", partOfSpeech); reflector.reflect(PartOfSpeechAttribute.class, "partOfSpeech (en)", en); }
@Override public void reflectWith(AttributeReflector reflector) { final List<String> pronunciations = getPronunciations(); List<String> enPronunciations = null; if (pronunciations != null) { final String[] p = new String[pronunciations.size()]; int i = 0; for (String kana : pronunciations) { p[i++] = ToStringUtil.getRomanization(kana); } enPronunciations = Arrays.asList(p); } reflector.reflect(PronunciationsAttribute.class, "pronunciations", pronunciations); reflector.reflect(PronunciationsAttribute.class, "pronunciations (en)", enPronunciations); }
@Override public void reflectWith(AttributeReflector reflector) { String conjugationalForm = getConjugationalForm(); String conjugationalType = getConjugationalType(); String enForm = conjugationalForm == null ? null : ToStringUtil.getConjFormTranslation(conjugationalForm); String enType = conjugationalType == null ? null : ToStringUtil.getConjTypeTranslation(conjugationalType); reflector.reflect(ConjugationAttribute.class, "conjugationalForm", conjugationalForm); reflector.reflect(ConjugationAttribute.class, "conjugationalForm (en)", enForm); reflector.reflect(ConjugationAttribute.class, "conjugationalType", conjugationalType); reflector.reflect(ConjugationAttribute.class, "conjugationalType (en)", enType); }
@Override public void reflectWith(AttributeReflector reflector) { fillBytesRef(); reflector.reflect(TermToBytesRefAttribute.class, "bytes", BytesRef.deepCopyOf(bytes)); reflector.reflect(NumericTermAttribute.class, "shift", shift); reflector.reflect(NumericTermAttribute.class, "rawValue", getRawValue()); reflector.reflect(NumericTermAttribute.class, "valueSize", valueSize); }
/** * other attribute extract object.<br/> * Extracted object group by AttributeClassName * * @param stream current TokenStream * @param includeAttributes filtering attributes * @param shortAttrName if true, return short attribute name * @return Nested Object : Map<attrClass, Map<key, value>> */ private Map<String, Map<String, Object>> extractExtendedAttributes(TokenStream stream, final Set<String> includeAttributes, final boolean shortAttrName) { final Map<String, Map<String, Object>> extendedAttributes = new TreeMap<>(); stream.reflectWith(new AttributeReflector() { @Override public void reflect(Class<? extends Attribute> attClass, String key, Object value) { if (CharTermAttribute.class.isAssignableFrom(attClass)) return; if (PositionIncrementAttribute.class.isAssignableFrom(attClass)) return; if (OffsetAttribute.class.isAssignableFrom(attClass)) return; if (TypeAttribute.class.isAssignableFrom(attClass)) return; if (includeAttributes == null || includeAttributes.isEmpty() || includeAttributes.contains(attClass.getSimpleName().toLowerCase())) { Map<String, Object> currentAttributes = extendedAttributes.get(attClass.getName()); if (currentAttributes == null) { currentAttributes = new HashMap<>(); } if (value instanceof BytesRef) { final BytesRef p = (BytesRef) value; value = p.toString(); } currentAttributes.put(key, value); if (shortAttrName) { extendedAttributes.put(attClass.getName().substring(attClass.getName().lastIndexOf(".")+1), currentAttributes); } else { extendedAttributes.put(attClass.getName(), currentAttributes); } } } }); return extendedAttributes; }
