spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vanzin <...@git.apache.org>
Subject [GitHub] spark pull request #22504: [SPARK-25118][Submit] Persist Driver Logs in Clie...
Date Fri, 26 Oct 2018 21:43:17 GMT
Github user vanzin commented on a diff in the pull request:

    https://github.com/apache/spark/pull/22504#discussion_r228673330
  
    --- Diff: core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
---
    @@ -413,6 +417,68 @@ class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter
with Matc
         }
       }
     
    +  test("driver log cleaner") {
    +    val firstFileModifiedTime = TimeUnit.SECONDS.toMillis(10)
    +    val secondFileModifiedTime = TimeUnit.SECONDS.toMillis(20)
    +    val maxAge = TimeUnit.SECONDS.toSeconds(40)
    +    val clock = new ManualClock(0)
    +    val testConf = new SparkConf()
    +    testConf.set("spark.history.fs.logDirectory",
    +      Utils.createTempDir(namePrefix = "eventLog").getAbsolutePath())
    +    testConf.set(DRIVER_LOG_DFS_DIR, testDir.getAbsolutePath())
    +    testConf.set(DRIVER_LOG_CLEANER_ENABLED, true)
    +    testConf.set(DRIVER_LOG_CLEANER_INTERVAL, maxAge / 4)
    +    testConf.set(MAX_DRIVER_LOG_AGE_S, maxAge)
    +    val provider = new FsHistoryProvider(testConf, clock)
    +
    +    val log1 = FileUtils.getFile(testDir, "1" + DriverLogger.DRIVER_LOG_FILE_SUFFIX)
    +    createEmptyFile(log1)
    +    val modTime1 = System.currentTimeMillis()
    +
    +    clock.setTime(modTime1 + firstFileModifiedTime)
    +    provider.cleanDriverLogs()
    +
    +    val log2 = FileUtils.getFile(testDir, "2" + DriverLogger.DRIVER_LOG_FILE_SUFFIX)
    +    createEmptyFile(log2)
    +    val log3 = FileUtils.getFile(testDir, "3" + DriverLogger.DRIVER_LOG_FILE_SUFFIX)
    +    createEmptyFile(log3)
    +    val modTime2 = System.currentTimeMillis()
    +
    +    clock.setTime(modTime1 + secondFileModifiedTime)
    +    provider.cleanDriverLogs()
    +
    +    // This should not trigger any cleanup
    +    provider.listing.view(classOf[LogInfo]).iterator().asScala.toSeq.size should be(3)
    +
    +    // Should trigger cleanup for first file but not second one
    +    clock.setTime(modTime1 + firstFileModifiedTime + TimeUnit.SECONDS.toMillis(maxAge)
+ 1)
    +    provider.cleanDriverLogs()
    +    provider.listing.view(classOf[LogInfo]).iterator().asScala.toSeq.size should be(2)
    +    assert(!log1.exists())
    +    assert(log2.exists())
    +    assert(log3.exists())
    +
    +    // Should cleanup the second file but not the third file, as filelength changed.
    +    val writer = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(log3)))
    --- End diff --
    
    `Files.write` is shorter and nicer.


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message