flink-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Averell <lvhu...@gmail.com>
Subject Re: Strange behaviour with checkpointing and custom FilePathFilter
Date Fri, 21 Sep 2018 02:50:07 GMT
Hi Vino,

I am using a custom FileInputFormat, but the mentioned problem only comes
when I try a custom FilePathFilter. 

My whole file for that custom FilePathFilter is quoted below.

Regarding enabling DEBUG, which classes/packages should I turn DEBUG on? as
I am afraid that turning DEBUG on at the global level would be too heavy.

Thanks and regards,
Averell


==================
import java.util.Date

import org.apache.flink.api.common.io.FilePathFilter
import org.apache.flink.core.fs.Path
import org.slf4j.LoggerFactory

object SdcFilePathFilter {
	private val TIME_FORMAT = new java.text.SimpleDateFormat("yyyyMMdd hhmm")
	private val LOG = LoggerFactory.getLogger(classOf[SdcFilePathFilter])
}

class SdcFilePathFilter(lookBackPeriod: Long, homePath: Path) extends
FilePathFilter {
	private val homeDepth = homePath.depth()

	override def filterPath(filePath: Path): Boolean = {
		filePath == null ||
				filePath.getName.startsWith(".") ||
				filePath.getName.startsWith("_") ||
				filePath.getName.contains(FilePathFilter.HADOOP_COPYING) ||
						!(filePath.getName.endsWith(".tar.gz") ||
									filePath.getName.matches("""^\d{8}$""") ||
									(filePath.getName.matches("""^\d{4}$""") &&
										
SdcFilePathFilter.TIME_FORMAT.parse(s"${filePath.getParent.getName}
${filePath.getName}").getTime <
													new Date().getTime - lookBackPeriod))
	}
}



--
Sent from: http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/

Mime
View raw message