From 0282be9a6a86ab78b11e2cd42c48a0fc98a7325b Mon Sep 17 00:00:00 2001 From: Ludger Peters Date: Mon, 20 Oct 2014 22:05:49 +0200 Subject: [PATCH] Added tests and ability to use a batch inserter instead to speed up the process of data generation. from the tests found under examples, The batch inserter took at least half of the time in comparison to the normal engine. --- src/main/java/org/neo4j/neode/Dataset.java | 10 ++- .../neode/statistics/GraphStatistics.java | 8 ++ src/test/java/org/neo4j/neode/test/Db.java | 18 ++++ .../test/MultipleRelsPerNodeExample.java | 30 +++++++ .../neode/test/SocialNetworkExample.java | 82 +++++++++++++++++++ 5 files changed, 146 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/neo4j/neode/Dataset.java b/src/main/java/org/neo4j/neode/Dataset.java index 4f8e527..eb92f2e 100644 --- a/src/main/java/org/neo4j/neode/Dataset.java +++ b/src/main/java/org/neo4j/neode/Dataset.java @@ -41,8 +41,14 @@ T execute( BatchCommand command ) public void end() { - log.write( String.format( "Store [%s]\n\nEnd [%s] %s\n", - ((GraphDatabaseAPI) db).getStoreDir(), description, elapsedTime( runStartTime ) ) ); + //this should be updated since it is going to be deprecated + //@deprecated This will be moved to internal packages in the next major release. + if(db instanceof GraphDatabaseAPI) + { + log.write(String.format("Store [%s]\n\nEnd [%s] %s\n", + ((GraphDatabaseAPI) db).getStoreDir(), description, elapsedTime(runStartTime))); + } + log.write( String.format( "End [%s] %s\n", description, elapsedTime( runStartTime ) ) ); } GraphDatabaseService db() diff --git a/src/main/java/org/neo4j/neode/statistics/GraphStatistics.java b/src/main/java/org/neo4j/neode/statistics/GraphStatistics.java index 5e2d045..a54aa12 100644 --- a/src/main/java/org/neo4j/neode/statistics/GraphStatistics.java +++ b/src/main/java/org/neo4j/neode/statistics/GraphStatistics.java @@ -6,7 +6,9 @@ import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; +import org.neo4j.kernel.GraphDatabaseAPI; import org.neo4j.tooling.GlobalGraphOperations; +import org.neo4j.unsafe.batchinsert.BatchInserterImpl; public class GraphStatistics { @@ -14,6 +16,12 @@ public class GraphStatistics public static GraphStatistics create( GraphDatabaseService db, String description, NodeLabelResolver nodeLabelResolver ) { + //if db is not the normal graph database api + //We need to find another way to do this since the GraphDatabaseAPI is /** @deprecated */ + if(!(db instanceof GraphDatabaseAPI)) + { + return null; + } try ( Transaction tx = db.beginTx() ) { GraphStatistics graphStatistics = new GraphStatistics( description, nodeLabelResolver ); diff --git a/src/test/java/org/neo4j/neode/test/Db.java b/src/test/java/org/neo4j/neode/test/Db.java index 8c666ba..23868ac 100644 --- a/src/test/java/org/neo4j/neode/test/Db.java +++ b/src/test/java/org/neo4j/neode/test/Db.java @@ -9,7 +9,9 @@ import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseFactory; +import org.neo4j.kernel.impl.util.FileUtils; import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.unsafe.batchinsert.BatchInserters; public final class Db { @@ -81,4 +83,20 @@ public interface WithSampleDataset } + public static GraphDatabaseService getBatchInserterDB(String location) + { + File file = new File(location); + if(file.exists() && file.isDirectory()) + { + try + { + FileUtils.deleteRecursively(file); + } + catch (IOException e) + { + e.printStackTrace(); + } + } + return BatchInserters.batchDatabase(location); + } } diff --git a/src/test/java/org/neo4j/neode/test/MultipleRelsPerNodeExample.java b/src/test/java/org/neo4j/neode/test/MultipleRelsPerNodeExample.java index 670e691..371a668 100644 --- a/src/test/java/org/neo4j/neode/test/MultipleRelsPerNodeExample.java +++ b/src/test/java/org/neo4j/neode/test/MultipleRelsPerNodeExample.java @@ -3,6 +3,7 @@ import org.junit.Test; import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.factory.GraphDatabaseFactory; import org.neo4j.neode.Dataset; import org.neo4j.neode.DatasetManager; import org.neo4j.neode.NodeCollection; @@ -46,4 +47,33 @@ public void buildMultipleRelsPerNode() db.shutdown(); } + + @Test + public void batchBuildMultipleRelsPerNode() + { + GraphDatabaseService db = Db.getBatchInserterDB("batch.db"); + DatasetManager dsm = new DatasetManager( db, SysOutLog.INSTANCE ); + Dataset dataset = dsm.newDataset( "Multiple Rels Per Node" ); + + NodeSpecification root = dsm.nodeSpecification( "root", property("name") ); + NodeSpecification leaf = dsm.nodeSpecification( "leaf", property( "price", integerRange( 1, 10 ) ) ); + + RelationshipSpecification connected_to = dsm.relationshipSpecification( "CONNECTED_TO", + property( "quantity", integerRange( 1, 5 ) ) ); + + NodeCollection roots = root.create( 1 ).update( dataset ); + + roots.createRelationshipsTo( TargetNodesStrategy.create( leaf ).numberOfTargetNodes( 2 ).relationship( + connected_to ).relationshipConstraints( Range.exactly( 2 ) ) ).updateNoReturn( dataset ); + + dataset.end(); + db.shutdown(); + db = new GraphDatabaseFactory().newEmbeddedDatabase("batch.db"); + //since we cant use graph statistics we have to end the dataset shut it down and open it up using graphFactory + GraphStatistics statistics = GraphStatistics.create( db, "Pricing Structure" ); + new AsciiDocFormatter( SysOutLog.INSTANCE ).describe( statistics ); + + db.shutdown(); + + } } diff --git a/src/test/java/org/neo4j/neode/test/SocialNetworkExample.java b/src/test/java/org/neo4j/neode/test/SocialNetworkExample.java index 05536aa..bdfcce9 100644 --- a/src/test/java/org/neo4j/neode/test/SocialNetworkExample.java +++ b/src/test/java/org/neo4j/neode/test/SocialNetworkExample.java @@ -5,6 +5,7 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Path; +import org.neo4j.graphdb.factory.GraphDatabaseFactory; import org.neo4j.graphdb.traversal.Evaluation; import org.neo4j.graphdb.traversal.Evaluator; import org.neo4j.graphdb.traversal.TraversalDescription; @@ -109,4 +110,85 @@ public Evaluation evaluate( Path path ) System.out.println( "Number of topics: " + topics.size() ); } + + @Test + public void buildSocialNetworkBatchInserter() throws Exception + { + String SocialNetwork = "socialNetwork.db"; + GraphDatabaseService db = Db.getBatchInserterDB(SocialNetwork); + DatasetManager dsm = new DatasetManager( db, SysOutLog.INSTANCE ); + + TraversalDescription findCompanyProjects = Traversal.description() + .depthFirst() + .relationships( withName( "WORKS_FOR" ), Direction.BOTH ) + .relationships( withName( "WORKED_ON" ), Direction.OUTGOING ) + .evaluator( new Evaluator() + { + @Override + public Evaluation evaluate( Path path ) + { + if ( path.lastRelationship() == null ) + { + return Evaluation.EXCLUDE_AND_CONTINUE; + } + if ( path.lastRelationship().isType( withName( "WORKED_ON" ) ) ) + { + return Evaluation.INCLUDE_AND_PRUNE; + } + return Evaluation.EXCLUDE_AND_CONTINUE; + } + } ); + + NodeSpecification user = dsm.nodeSpecification( "user", property( "name" ) ); + NodeSpecification topic = dsm.nodeSpecification( "topic", property( "label" ) ); + NodeSpecification company = dsm.nodeSpecification( "company", property( "name" ) ); + NodeSpecification project = dsm.nodeSpecification( "project", property( "name" ) ); + + RelationshipSpecification interested_in = dsm.relationshipSpecification( "INTERESTED_IN" ); + RelationshipSpecification works_for = dsm.relationshipSpecification( "WORKS_FOR" ); + RelationshipSpecification worked_on = dsm.relationshipSpecification( "WORKED_ON" ); + + Dataset dataset = dsm.newDataset( "Social network example" ); + + NodeCollection users = user.create( 10 ).update( dataset ); + + NodeCollection topics = users.createRelationshipsTo( + getOrCreate( topic, 10, normalDistribution() ) + .numberOfTargetNodes( minMax( 1, 3 ) ) + .relationship( interested_in ) + .exactlyOneRelationship() ) + .update( dataset ); + + users.createRelationshipsTo( + getOrCreate( company, 2, flatDistribution() ) + .numberOfTargetNodes( 1 ) + .relationship( works_for ) + .exactlyOneRelationship() ) + .updateNoReturn( dataset ); + + NodeCollection allProjects = users.createRelationshipsTo( + queryBasedGetOrCreate( project, traversal( findCompanyProjects ), 1.2 ) + .numberOfTargetNodes( minMax( 1, 3 ) ) + .relationship( worked_on ) + .exactlyOneRelationship() ) + .update( dataset ); + + users.approxPercentage( 30 ).createRelationshipsTo( + getExisting( allProjects ) + .numberOfTargetNodes( minMax( 1, 2 ) ) + .relationship( worked_on ) + .relationshipConstraints( RelationshipUniqueness.SINGLE_DIRECTION ) ) + .updateNoReturn( dataset ); + + dataset.end(); + + db.shutdown(); + db = new GraphDatabaseFactory().newEmbeddedDatabase(SocialNetwork); + GraphStatistics statistics = GraphStatistics.create( db, "Social Network" ); + new AsciiDocFormatter( SysOutLog.INSTANCE ).describe( statistics ); + + db.shutdown(); + + System.out.println( "Number of topics: " + topics.size() ); + } }