Removed Graph Caching To Prevent Write Failures Propagating

plume-oss · Aug 21, 2024 · 6eed1e4 · 6eed1e4
1 parent 4754a9c
commit 6eed1e4
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 41 deletions.
diff --git a/runBenchmarks.sc b/runBenchmarks.sc
@@ -38,7 +38,8 @@ val drivers = Seq("overflowdb", "tinkergraph", "neo4j-embedded")
   }
 
   println("[info] Available projects:")
-  val projects = Files.list(datasetDir).filter(_.toString.endsWith(".jar")).toList.asScala.toList
+  val projects =
+    Files.list(datasetDir).filter(_.toString.endsWith(".jar")).toList.asScala.sortBy(_.toFile.length()).toList
   projects.foreach(p => println(s" - ${p.getFileName.toString}"))
 
   println("[info] Drivers to be benchmarked:")
@@ -154,15 +155,24 @@ def runAndMonitorBenchmarkProcess(cmd: String, driver: String, writeOutputFile:
     }
     if (!outputPath.toFile.exists() && storageLoc.exists()) {
       val size = getFileSize(storageLoc)
+      println(s"${storageLoc.getAbsolutePath} is $size bytes large")
       outputPath.toFile.createIfNotExists
       Files.writeString(outputPath, size.toString)
-      storageLoc.delete()
+      // clear storage
+      deleteFileOrDir(storageLoc)
     }
   }
 
 }
 
-def getFileSize(f: File): Long = {
+def deleteFileOrDir(file: File): Unit = {
+  Option(file.listFiles).foreach { contents =>
+    contents.filterNot(f => Files.isSymbolicLink(f.toPath)).foreach(deleteFileOrDir)
+  }
+  file.delete
+}
+
+def getFileSize(f: File) = {
   if (f.isFile) {
     f.length()
   } else {

diff --git a/src/main/scala/com/github/plume/oss/Benchmark.scala b/src/main/scala/com/github/plume/oss/Benchmark.scala
@@ -8,7 +8,7 @@ import com.github.plume.oss.benchmarking.{
   OverflowDbReadBenchmark,
   TinkerGraphReadBenchmark
 }
-import com.github.plume.oss.drivers.{IDriver, TinkerGraphDriver}
+import com.github.plume.oss.drivers.IDriver
 import org.cache2k.benchmark.jmh.{HeapProfiler, LinuxVmProfiler}
 import org.openjdk.jmh.annotations.Mode
 import org.openjdk.jmh.runner.Runner
@@ -80,31 +80,19 @@ object Benchmark {
     case READ, WRITE
   }
 
-  def initializeDriverAndInputDir(configStr: String, useCachedGraph: Boolean): (IDriver, PlumeConfig) = {
+  def initializeDriverAndInputDir(configStr: String): (IDriver, PlumeConfig) = {
     val config = if (!configStr.isBlank) read[PlumeConfig](configStr) else PlumeConfig()
-    if (!useCachedGraph) {
-      config.dbConfig match {
-        case OverflowDbConfig(storageLocation, _, _) if !useCachedGraph =>
-          File(storageLocation).delete(swallowIOExceptions = true)
-        case TinkerGraphConfig(Some(importPath), _) if !useCachedGraph =>
-          File(importPath).delete(swallowIOExceptions = true)
-        case Neo4jEmbeddedConfig(_, databaseDir, _) /*if !useCachedGraph */ =>
-          File(databaseDir).delete(swallowIOExceptions = true)
-        case _ =>
-      }
+    config.dbConfig match {
+      case OverflowDbConfig(storageLocation, _, _) =>
+        File(storageLocation).delete(swallowIOExceptions = true)
+      case TinkerGraphConfig(Some(importPath), _) =>
+        File(importPath).delete(swallowIOExceptions = true)
+      case Neo4jEmbeddedConfig(_, databaseDir, _) =>
+        File(databaseDir).delete(swallowIOExceptions = true)
+      case _ =>
     }
 
-    val driver = if (useCachedGraph) {
-      config.dbConfig match {
-        case TinkerGraphConfig(Some(importPath), _) if File(importPath).exists =>
-          val driver = config.dbConfig.toDriver.asInstanceOf[TinkerGraphDriver]
-          driver.importGraph(importPath)
-          driver
-        case _ => config.dbConfig.toDriver
-      }
-    } else {
-      config.dbConfig.toDriver
-    }
+    val driver = config.dbConfig.toDriver
 
     driver -> config
   }

diff --git a/src/main/scala/com/github/plume/oss/benchmarking/GraphReadBenchmark.scala b/src/main/scala/com/github/plume/oss/benchmarking/GraphReadBenchmark.scala
@@ -53,16 +53,10 @@ trait GraphReadBenchmark {
   }
 
   protected def setupBenchmark(params: BenchmarkParams): Unit = {
-    val (driver_, config_) = oss.Benchmark.initializeDriverAndInputDir(configStr, useCachedGraph = true)
+    val (driver_, config_) = oss.Benchmark.initializeDriverAndInputDir(configStr)
     driver = driver_
     config = config_
-    if (driver.propertyFromNodes(NodeTypes.FILE, PropertyNames.NAME).isEmpty) {
-      JimpleAst2Database(driver).createAst(Config().withInputPath(config_.inputDir))
-      config.dbConfig match {
-        case TinkerGraphConfig(_, Some(exportPath)) => driver.asInstanceOf[TinkerGraphDriver].exportGraph(exportPath)
-        case _                                      =>
-      }
-    }
+    JimpleAst2Database(driver).createAst(Config().withInputPath(config_.inputDir))
   }
 
   protected def setupAstDfs(): Array[Long]

diff --git a/src/main/scala/com/github/plume/oss/benchmarking/GraphWriteBenchmark.scala b/src/main/scala/com/github/plume/oss/benchmarking/GraphWriteBenchmark.scala
@@ -18,14 +18,14 @@ import scala.compiletime.uninitialized
 class GraphWriteBenchmark {
 
   @Param(Array(""))
-  var configStr: String        = ""
+  var configStr: String           = ""
   private var config: PlumeConfig = uninitialized
-  private var driver: IDriver  = uninitialized
-  private var inputDir: String = uninitialized
+  private var driver: IDriver     = uninitialized
+  private var inputDir: String    = uninitialized
 
   @Setup
   def setupBenchmark(params: BenchmarkParams): Unit = {
-    val (driver_, config_) = oss.Benchmark.initializeDriverAndInputDir(configStr, useCachedGraph = false)
+    val (driver_, config_) = oss.Benchmark.initializeDriverAndInputDir(configStr)
     driver = driver_
     config = config_
     inputDir = config.inputDir
@@ -48,9 +48,10 @@ class GraphWriteBenchmark {
   @TearDown
   def cleanupBenchmark(): Unit = {
     driver match {
-      case x: TinkerGraphDriver => config.dbConfig.asInstanceOf[TinkerGraphConfig].exportPath.foreach { path =>
-        x.exportGraph(path)
-      }
+      case x: TinkerGraphDriver =>
+        config.dbConfig.asInstanceOf[TinkerGraphConfig].exportPath.foreach { path =>
+          x.exportGraph(path)
+        }
       case _ =>
     }
     driver.close()