Re: Strange behaviour with checkpointing and custom FilePathFilter

[Averell]

Hi Vino,

I am using a custom FileInputFormat, but the mentioned problem only comes
when I try a custom FilePathFilter. 

My whole file for that custom FilePathFilter is quoted below.

Regarding enabling DEBUG, which classes/packages should I turn DEBUG on? as
I am afraid that turning DEBUG on at the global level would be too heavy.

Thanks and regards,
Averell


==================
import java.util.Date

import org.apache.flink.api.common.io.FilePathFilter
import org.apache.flink.core.fs.Path
import org.slf4j.LoggerFactory

object SdcFilePathFilter {
        private val TIME_FORMAT = new java.text.SimpleDateFormat("yyyyMMdd 
hhmm")
        private val LOG = LoggerFactory.getLogger(classOf[SdcFilePathFilter])
}

class SdcFilePathFilter(lookBackPeriod: Long, homePath: Path) extends
FilePathFilter {
        private val homeDepth = homePath.depth()

        override def filterPath(filePath: Path): Boolean = {
                filePath == null ||
                                filePath.getName.startsWith(".") ||
                                filePath.getName.startsWith("_") ||
                                
filePath.getName.contains(FilePathFilter.HADOOP_COPYING) ||
                                                
!(filePath.getName.endsWith(".tar.gz") ||
                                                                        
filePath.getName.matches("""^\d{8}$""") ||
                                                                        
(filePath.getName.matches("""^\d{4}$""") &&
                                                                                
SdcFilePathFilter.TIME_FORMAT.parse(s"${filePath.getParent.getName}
${filePath.getName}").getTime <
                                                                                
                        new Date().getTime - lookBackPeriod))
        }
}



--
Sent from: http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/