public Long getTypedNodeTokenCount(String type, String pattern, Boolean sensitive) { Long frequency = (long) 0; try ( Transaction ignored = database.beginTx(); ) { IndexManager index = database.index(); Index<Node> nodeIndex = index.forNodes(type); for (Node match : nodeIndex.get("label", pattern)) { if (!sensitive || ((String) match.getProperty("label")).equals(pattern)) { Object count = match.getProperty("token_count"); if (count instanceof Integer) frequency = frequency + new Long((Integer) count); else if (count instanceof Long) frequency = frequency + (Long) count; } } } return frequency; }
protected Response getMetadataLabel(final String label) { StreamingOutput stream = new StreamingOutput() { @Override public void write( OutputStream os ) throws IOException, WebApplicationException { try ( Transaction tx = database.beginTx() ) { JsonGenerator jg = objectMapper.getFactory().createGenerator(os, JsonEncoding.UTF8); IndexManager index = database.index(); Index<Node> metadata = index.forNodes("Metadatum"); jg.writeStartArray(); for ( Node metadatum : metadata.query( "label:"+label ) ) { executor.writeField(metadatum, jg); } jg.writeEndArray(); jg.flush(); tx.success(); } } }; return Response.ok().entity( stream ).type( MediaType.APPLICATION_JSON ).build(); }
protected Response updateMetadataLabel(final String label, final String property, final String value) { StreamingOutput stream = new StreamingOutput() { @Override public void write( OutputStream os ) throws IOException, WebApplicationException { try ( Transaction tx = database.beginTx() ) { JsonGenerator jg = objectMapper.getFactory().createGenerator(os, JsonEncoding.UTF8); IndexManager index = database.index(); Index<Node> metadata = index.forNodes("Metadatum"); for ( Node metadatum : metadata.query( "label:"+label ) ) { if (property.equals("explorable") || property.equals("searchable")) metadatum.setProperty(property, Boolean.valueOf(value)); else metadatum.setProperty(property, value); } jg.writeString("Updated "+label); jg.flush(); tx.success(); } } }; return Response.ok().entity( stream ).type( MediaType.APPLICATION_JSON ).build(); }
protected Response getMetadataGroupKey(final String group, final String key) { StreamingOutput stream = new StreamingOutput() { @Override public void write( OutputStream os ) throws IOException, WebApplicationException { try ( Transaction tx = database.beginTx() ) { JsonGenerator jg = objectMapper.getFactory().createGenerator(os, JsonEncoding.UTF8); IndexManager index = database.index(); Index<Node> metadata = index.forNodes("Metadatum"); jg.writeStartArray(); for ( Node metadatum : metadata.query( "group:"+group+" AND key:"+key ) ) { executor.writeField(metadatum, jg); } jg.writeEndArray(); jg.flush(); tx.success(); } } }; return Response.ok().entity( stream ).type( MediaType.APPLICATION_JSON ).build(); }
@Test public void deleteIndex() { GraphDatabaseService graphDb = new TestGraphDatabaseFactory().newImpermanentDatabase(); try ( Transaction tx = graphDb.beginTx() ) { // START SNIPPET: delete IndexManager index = graphDb.index(); Index<Node> actors = index.forNodes( "actors" ); actors.delete(); // END SNIPPET: delete tx.success(); } assertFalse( indexExists( graphDb ) ); graphDb.shutdown(); }
@Test public void fulltext() { // START SNIPPET: fulltext IndexManager index = graphDb.index(); Index<Node> fulltextMovies = index.forNodes( "movies-fulltext", MapUtil.stringMap( IndexManager.PROVIDER, "lucene", "type", "fulltext" ) ); // END SNIPPET: fulltext Index<Node> movies = index.forNodes( "movies" ); Node theMatrix = movies.get( "title", "The Matrix" ).getSingle(); Node theMatrixReloaded = movies.get( "title", "The Matrix Reloaded" ).getSingle(); // START SNIPPET: fulltext fulltextMovies.add( theMatrix, "title", "The Matrix" ); fulltextMovies.add( theMatrixReloaded, "title", "The Matrix Reloaded" ); // search in the fulltext index Node found = fulltextMovies.query( "title", "reloAdEd" ).getSingle(); // END SNIPPET: fulltext assertEquals( theMatrixReloaded, found ); }
@Test public void testCanIndexRelationships() { BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProviderNewImpl( inserter ); BatchInserterIndex edgesIndex = indexProvider.relationshipIndex( "edgeIndex", stringMap( IndexManager.PROVIDER, "lucene", "type", "exact" ) ); long nodeId1 = inserter.createNode( map( "ID", "1" ) ); long nodeId2 = inserter.createNode( map( "ID", "2" ) ); long relationshipId = inserter.createRelationship( nodeId1, nodeId2, EdgeType.KNOWS, null ); edgesIndex.add( relationshipId, map( "EDGE_TYPE", EdgeType.KNOWS.name() ) ); edgesIndex.flush(); assertEquals( String.format( "Should return relationship id" ), new Long( relationshipId ), edgesIndex.query( "EDGE_TYPE", EdgeType.KNOWS.name() ).getSingle() ); indexProvider.shutdown(); }
public Neo4JEdgeIndex(String name, Neo4JDatabase neo4jDatabase) { graph = neo4jDatabase; this.name = name; if (neo4jDatabase.getGraph() != null) index = neo4jDatabase.getIndexer().forRelationships( name, MapUtil.stringMap(IndexManager.PROVIDER, "lucene", "type", "exact")); else batchIndex = neo4jDatabase.getBatchIndexer().relationshipIndex( name, MapUtil.stringMap(IndexManager.PROVIDER, "lucene", "type", "exact")); }
private WikiNerGraphConnector(String dbDir) { graphDb = new GraphDatabaseFactory().newEmbeddedDatabase(dbDir); IndexDefinition indexDefinition; Schema schema; Transaction tx = graphDb.beginTx(); schema = graphDb.schema(); IndexManager index = graphDb.index(); entities = index.forNodes(entityLabel.name()); cities = index.forNodes(cityLabel.name()); try{ indexDefinition = schema.indexFor( entityLabel ).on( "nameType" ).create(); }catch(Exception e){ } tx.success(); tx.close(); }
public static Node getOrCreateUserNode(final GraphDatabaseService graphDb, final String userID) { IndexManager index = graphDb.index(); Index<Node> usersIndex = index.forNodes("users"); Node node = usersIndex.get("node_id", userID).getSingle(); if (node == null) { node = graphDb.createNode(DynamicLabel.label("User")); node.setProperty("node_id", userID); usersIndex.add(node, "node_id", userID); } return node; }
public static Node getOrCreateItemNode(final GraphDatabaseService graphDb, final String itemID) { IndexManager index = graphDb.index(); Index<Node> usersIndex = index.forNodes("items"); Node node = usersIndex.get("node_id", itemID).getSingle(); if (node == null) { node = graphDb.createNode(DynamicLabel.label("Item")); node.setProperty("node_id", itemID); usersIndex.add(node, "node_id", itemID); } return node; }
@Override public void addItem(int itemId) { Node node = getNodeFromIndex(id); int freq = 1; //first check if the item has already been added String itemAsStr = ITEM_ID_PREFIX + Integer.toString(itemId); if(node.hasProperty(itemAsStr)){ Integer prevFreq = (Integer) node.getProperty(itemAsStr); freq = prevFreq +1; } else{ //add to index only once //add to lucene index IndexManager IM = Utilities.recordDB.index(); Index<Node> itemIdx = IM.forNodes(ITEM_INDEX_NAME); itemIdx.add(node, itemAsStr, "1"); } node.setProperty(itemAsStr, freq); }
public void addItem(int itemId) { Node node = getNodeFromIndex(id); int freq = 1; //first check if the item has already been added String itemAsStr = ITEM_ID_PREFIX + Integer.toString(itemId); if(node.hasProperty(itemAsStr)){ Integer prevFreq = (Integer) node.getProperty(itemAsStr); freq = prevFreq +1; } else{ //add to index only once //add to lucene index IndexManager IM = Utilities.recordDB.index(); Index<Node> itemIdx = IM.forNodes(ITEM_INDEX_NAME); itemIdx.add(node, itemAsStr, "1"); } node.setProperty(itemAsStr, freq); }
@Override public void createNodeIndex(String name, ParcelableError err) throws RemoteException { try { checkCallerHasWritePermission(); resumeTrx(); try { IndexManager index = mDb.index(); index.forNodes(name); // this will create the index } finally { suspendCurrentTrx("createNodeIndex"); } } catch (Exception e) { Log.e(TAG, "Failed to create/access node index '" + name + "'", e); err.setError(Errors.TRANSACTION, e.getMessage()); } }
@Override public boolean nodeIndexExists(String name, ParcelableError err) throws RemoteException { try { resumeTrxIfExists(); try { IndexManager index = mDb.index(); return index.existsForNodes(name); } finally { suspendCurrentTrx("nodeIndexExists"); } } catch (Exception e) { Log.e(TAG, "Failed to determine if node index '" + name + "' exists", e); err.setError(Errors.TRANSACTION, e.getMessage()); return false; // dummy return value } }
@Override public void deleteNodeIndex(String name, ParcelableError err) throws RemoteException { try { checkCallerHasWritePermission(); resumeTrx(); try { IndexManager index = mDb.index(); index.forNodes(name); // this will create the index } finally { suspendCurrentTrx("deleteNodeIndex"); } } catch (Exception e) { Log.e(TAG, "Failed to delete node index '" + name + "'", e); err.setError(Errors.TRANSACTION, e.getMessage()); } }
@Override public void execute(TinkerPopGraphManager graphWrapper) throws IOException { GraphDatabaseService service = graphWrapper.getGraphDatabase(); IndexManager indexManager = service.index(); try (Transaction tx = graphWrapper.getGraph().tx()) { clearIndices(indexManager); tx.commit(); } try (Transaction tx = graphWrapper.getGraph().tx()) { if (!tx.isOpen()) { tx.open(); } constructIndices(indexManager); fillVertexIndices(graphWrapper, indexManager); fillEdgeIndex(service, indexManager); tx.commit(); } }
private void fillVertexIndices(TinkerPopGraphManager graphWrapper, IndexManager indexManager) { ObjectMapper mapper = new ObjectMapper(); GraphTraversalSource traversalSource = graphWrapper.getGraph().traversal(); Collection wwpersonCollection = vres.getCollection("wwpersons").orElse(null); graphWrapper.getGraphDatabase().getAllNodes().stream() .filter(node -> node.hasProperty("tim_id")) .filter(this::isLatest) .forEach(node -> { updateIdIndex(indexManager, node); if (!isDeleted(node)) { updateQuicksearchIndex(vres, indexManager, mapper, node, wwpersonCollection, traversalSource); } }); }
private void updateQuicksearchIndex(Vres vres, IndexManager indexManager, ObjectMapper mapper, Node node, Collection wwpersonCollection, GraphTraversalSource traversalSource) { for (String type : getEntityTypes(node, mapper)) { if (!type.equals("relationtype")) { Optional<Collection> collection = vres.getCollectionForType(type); long id = node.getId(); if (collection.isPresent()) { String displayName; if (type.equals("wwdocument")) { displayName = getWwDocumentsQuickSearchValue(collection.get(), wwpersonCollection, id, traversalSource ); } else { displayName = getGenericQuickSearchValue(collection.get(), id, traversalSource); } indexManager.forNodes(collection.get().getCollectionName()).add(node, "quickSearch", displayName); } else { LOG.error("Could not find collection for " + type + " at vertex " + id); } } } }
@GET @Path("/tags/{label}") public Response getPosTagByLabel(@PathParam("label") final String label) { StreamingOutput stream = new StreamingOutput() { @Override public void write( OutputStream os ) throws IOException, WebApplicationException { try ( Transaction ignored = database.beginTx() ) { JsonGenerator jg = objectMapper.getFactory().createGenerator(os, JsonEncoding.UTF8); IndexManager index = database.index(); Index<Node> postags = index.forNodes("PosTag"); IndexHits<Node> hits = postags.get( "label", label ); Node postag = hits.getSingle(); jg.writeStartObject(); for (String field : postag.getPropertyKeys()) { Object value = postag.getProperty(field); if (field.contains("_count") && value instanceof Integer) jg.writeNumberField(field, (Integer) value); else if (field.contains("_count") && value instanceof Long) jg.writeNumberField(field, (Long) value); else jg.writeStringField(field, (String) postag.getProperty(field)); } jg.writeEndObject(); jg.flush(); jg.close(); } } }; return Response.ok().entity( stream ).type( MediaType.APPLICATION_JSON ).build(); }
@GET @Path("/heads/{label}") public Response getPosHeadByLabel(@PathParam("label") final String label) { StreamingOutput stream = new StreamingOutput() { @Override public void write( OutputStream os ) throws IOException, WebApplicationException { try ( Transaction ignored = database.beginTx() ) { JsonGenerator jg = objectMapper.getFactory().createGenerator(os, JsonEncoding.UTF8); IndexManager index = database.index(); Index<Node> postags = index.forNodes("PosHead"); IndexHits<Node> hits = postags.get( "label", label ); Node poshead = hits.getSingle(); jg.writeStartObject(); for (String field : poshead.getPropertyKeys()) { Object value = poshead.getProperty(field); if (field.contains("_count") && value instanceof Integer) jg.writeNumberField(field, (Integer) value); else if (field.contains("_count") && value instanceof Long) jg.writeNumberField(field, (Long) value); else jg.writeStringField(field, (String) poshead.getProperty(field)); } jg.writeEndObject(); jg.flush(); jg.close(); } } }; return Response.ok().entity( stream ).type( MediaType.APPLICATION_JSON ).build(); }
protected Response getMetadataGroupKeyValueDocuments(final String group, final String key, final String value) { StreamingOutput stream = new StreamingOutput() { @Override public void write( OutputStream os ) throws IOException, WebApplicationException { try ( Transaction tx = database.beginTx() ) { JsonGenerator jg = objectMapper.getFactory().createGenerator(os, JsonEncoding.UTF8); IndexManager index = database.index(); Index<Relationship> metadataValues = index.forRelationships("HAS_METADATUM"); List<String> documentsSeen = new ArrayList<String>(); jg.writeStartArray(); System.out.println("VALUE: "+value); for (Relationship hasMetadatum : metadataValues.get("value", value)) { Node metadatum = hasMetadatum.getEndNode(); String mGroup = (String) metadatum.getProperty("group"); String mKey = (String) metadatum.getProperty("key"); if (mGroup.equals(group) && mKey.equals(key)) { Node document = hasMetadatum.getStartNode(); String xmlid = (String) document.getProperty("xmlid"); if (!documentsSeen.contains(xmlid)) { jg.writeString(xmlid); documentsSeen.add(xmlid); } } } jg.writeEndArray(); jg.flush(); tx.success(); } } }; return Response.ok().entity( stream ).type( MediaType.APPLICATION_JSON ).build(); }
private void assertIsLuceneIndex( GraphDatabaseService db, Index<T> index ) { Map<String, String> config = db.index().getConfiguration( index ); if ( !config.get( IndexManager.PROVIDER ).equals( "lucene" ) ) // Not so hard coded please { throw new IllegalArgumentException( index + " isn't a Lucene index" ); } }
@Test public void checkIfIndexExists() { // START SNIPPET: checkIfExists IndexManager index = graphDb.index(); boolean indexExists = index.existsForNodes( "actors" ); // END SNIPPET: checkIfExists assertTrue( indexExists ); }
@Test public void update() { IndexManager index = graphDb.index(); Index<Node> actors = index.forNodes( "actors" ); // START SNIPPET: update // create a node with a property // so we have something to update later on Node fishburn = graphDb.createNode(); fishburn.setProperty( "name", "Fishburn" ); // index it actors.add( fishburn, "name", fishburn.getProperty( "name" ) ); // END SNIPPET: update Node node = actors.get( "name", "Fishburn" ).getSingle(); assertEquals( fishburn, node ); // START SNIPPET: update // update the index entry // when the property value changes actors.remove( fishburn, "name", fishburn.getProperty( "name" ) ); fishburn.setProperty( "name", "Laurence Fishburn" ); actors.add( fishburn, "name", fishburn.getProperty( "name" ) ); // END SNIPPET: update node = actors.get( "name", "Fishburn" ).getSingle(); assertEquals( null, node ); node = actors.get( "name", "Laurence Fishburn" ).getSingle(); assertEquals( fishburn, node ); }
@Test public void makeSureCustomAnalyzerCanBeUsed() { CustomAnalyzer.called = false; Index<Node> index = nodeIndex( MapUtil.stringMap( IndexManager.PROVIDER, "lucene", "analyzer", org.neo4j.index.impl.lucene.CustomAnalyzer.class.getName(), "to_lower_case", "true" ) ); Node node = graphDb.createNode(); String key = "name"; String value = "The value"; index.add( node, key, value ); restartTx(); assertTrue( CustomAnalyzer.called ); assertThat( index.query( key, "[A TO Z]" ), contains( node ) ); }
@Test public void makeSureCustomAnalyzerCanBeUsed2() { CustomAnalyzer.called = false; Index<Node> index = nodeIndex( "w-custom-analyzer-2", MapUtil.stringMap( IndexManager.PROVIDER, "lucene", "analyzer", org.neo4j.index.impl.lucene.CustomAnalyzer.class.getName(), "to_lower_case", "true", "type", "fulltext" ) ); Node node = graphDb.createNode(); String key = "name"; String value = "The value"; index.add( node, key, value ); restartTx(); assertTrue( CustomAnalyzer.called ); assertThat( index.query( key, "[A TO Z]" ), contains( node ) ); }
@Test public void makeSureYouCanGetEntityTypeFromIndex() { Index<Node> nodeIndex = nodeIndex( MapUtil.stringMap( IndexManager.PROVIDER, "lucene", "type", "exact" ) ); Index<Relationship> relIndex = relationshipIndex( MapUtil.stringMap( IndexManager.PROVIDER, "lucene", "type", "exact" ) ); assertEquals( Node.class, nodeIndex.getEntityType() ); assertEquals( Relationship.class, relIndex.getEntityType() ); }
@Test public void makeSureSlightDifferencesInIndexConfigCanBeSupplied() { Map<String, String> config = MapUtil.stringMap( IndexManager.PROVIDER, "lucene", "type", "fulltext" ); String name = currentIndexName(); nodeIndex( name, config ); nodeIndex( name, MapUtil.stringMap( new HashMap<>( config ), "to_lower_case", "true" ) ); try { nodeIndex( name, MapUtil.stringMap( new HashMap<>( config ), "to_lower_case", "false" ) ); fail( "Shouldn't be able to get index with these kinds of differences in config" ); } catch ( IllegalArgumentException e ) { /* */ } nodeIndex( name, MapUtil.stringMap( new HashMap<>( config ), "whatever", "something" ) ); }
@Test public void testSimilarity() { Index<Node> index = nodeIndex( MapUtil.stringMap( IndexManager.PROVIDER, "lucene", "type", "fulltext", "similarity", DefaultSimilarity.class.getName() ) ); Node node = graphDb.createNode(); index.add( node, "key", "value" ); restartTx(); assertContains( index.get( "key", "value" ), node ); }
private void removeProvidersFromIndexDbFile( File storeDir ) { IndexConfigStore indexStore = new IndexConfigStore( storeDir, new DefaultFileSystemAbstraction() ); for ( Class<? extends PropertyContainer> cls : new Class[] {Node.class, Relationship.class} ) { for ( String name : indexStore.getNames( cls ) ) { Map<String, String> config = indexStore.get( cls, name ); config = new HashMap<>( config ); config.remove( IndexManager.PROVIDER ); indexStore.set( Node.class, name, config ); } } }
public Neo4JNodeIndex(String name, Neo4JDatabase neo4jDatabase) { graph = neo4jDatabase; this.name = name; if (neo4jDatabase.getGraph() != null) index = neo4jDatabase.getIndexer().forNodes(name, MapUtil.stringMap(IndexManager.PROVIDER, "lucene", "type", "exact")); else batchIndex = neo4jDatabase.getBatchIndexer().nodeIndex(name, MapUtil.stringMap(IndexManager.PROVIDER, "lucene", "type", "exact")); }
public static Node getUserNode(final GraphDatabaseService graphDb, final String userID) throws ObelixNodeNotFoundException { IndexManager index = graphDb.index(); Index<Node> users = index.forNodes("users"); Node node = users.get("node_id", userID).getSingle(); if (node == null) { throw new ObelixNodeNotFoundException(); } return node; }
public DBRecord(int id, String recordStr){ this.id = id; //check id this record already exists in the DB Node node = getNodeFromIndex(id); if(node != null){ return; //record already exists in the DB } node = Utilities.recordDB.createNode(); node.setProperty(ID_PROP_NAME, id); node.setProperty(REC_STR_PROP_NAME, recordStr); //add to node index IndexManager IM = Utilities.recordDB.index(); Index<Node> nodeIdx = IM.forNodes(NODE_INDEX_NAME); nodeIdx.add(node, ID_PROP_NAME, id); }
private Node getNodeFromIndex(int id){ if(recordNode != null) return recordNode; IndexManager IM = Utilities.recordDB.index(); Index<Node> nodeIdx = IM.forNodes(NODE_INDEX_NAME); IndexHits<Node> hits = nodeIdx.get( "id", id ); recordNode = hits.getSingle(); return recordNode; }
private static List<IFRecord> getItemsetSupportDB(List<Integer> items, GraphDatabaseService recordsDB) { long start = System.currentTimeMillis(); List<IFRecord> retval = new LinkedList<IFRecord>(); StringBuilder queryBuilder = new StringBuilder(); boolean first = true; for (Integer itemId : items) { if (!first) { queryBuilder.append("AND "); } else { first = false; } queryBuilder.append(DBRecord.ITEM_ID_PREFIX).append( Integer.toString(itemId)).append(":1").append(" "); } // System.out.println("DEBUG: the query: " + queryBuilder.toString()); IndexManager IM = recordsDB.index(); Index<Node> recordIndex = IM.forNodes(DBRecord.ITEM_INDEX_NAME); QueryContext CQ = new QueryContext(queryBuilder.toString().trim()) .defaultOperator(Operator.AND); IndexHits<Node> support = recordIndex.query(CQ); for (Node recordNode : support) { retval.add(new DBRecord(recordNode)); } // System.out.println("DEBUG: supportsize: " + retval.size()); time_in_supp_calc.addAndGet(System.currentTimeMillis() - start); return retval; }
@Override public void start() throws Throwable{ IndexManager indexManager = this.gdb.index(); if(setupAutoIndexing){ setupTimestampIndexing(indexManager.getNodeAutoIndexer()); setupTimestampIndexing(indexManager.getRelationshipAutoIndexer()); } this.gdb.registerTransactionEventHandler(new TimestampTransactionEventHandler<String>(this.addCreated, this.customPropertyHandlers)); }