@Override public void reset() throws IOException { updateUserDictionary(); if (dictionaryTimestamp > tokenizerTimestamp) { if (VERBOSE) { System.out.println("Update KuromojiTokenizer (" + tokenizerTimestamp + "," + dictionaryTimestamp + ")"); } if (userDictionary != null) { try { tokenizerTimestamp = dictionaryTimestamp; userDictionaryField.set(tokenizer, userDictionary); final TokenInfoFST userFst = userDictionary.getFST(); userFSTField.set(tokenizer, userFst); userFSTReaderField.set(tokenizer, userFst.getBytesReader()); @SuppressWarnings("unchecked") final EnumMap<Type, Dictionary> dictionaryMap = (EnumMap<Type, Dictionary>) dictionaryMapField.get(tokenizer); dictionaryMap.put(Type.USER, userDictionary); } catch (final Exception e) { throw new IllegalStateException( "Failed to update the tokenizer.", e); } } } final Reader inputPending = getInputPending(); if (inputPending != ILLEGAL_STATE_READER) { tokenizer.setReader(inputPending); } tokenizer.reset(); }
public Token(int wordId, char[] surfaceForm, int offset, int length, Type type, int position, Dictionary dictionary) { this.wordId = wordId; this.surfaceForm = surfaceForm; this.offset = offset; this.length = length; this.type = type; this.position = position; this.dictionary = dictionary; }
/** * Returns true if this token is known word * @return true if this token is in standard dictionary. false if not. */ public boolean isKnown() { return type == Type.KNOWN; }
/** * Returns true if this token is unknown word * @return true if this token is unknown word. false if not. */ public boolean isUnknown() { return type == Type.UNKNOWN; }
/** * Returns true if this token is defined in user dictionary * @return true if this token is in user dictionary. false if not. */ public boolean isUser() { return type == Type.USER; }