Get directory content faster with many files Tag(s): IO
With Java 7, there is a new possibility to handle directory with a large amount of file (say more than 10 000 files). With the previous JDK versions, File.listFiles() returns a File array. The performance is not so good since you have to wait for the array to be completed before starting the processing. This is especially bad if you need to work only on subset of the content. With Java 7, java.nio offers an alternative with a huge gain in performance.
In this HowTo, we want to process only the 10 first files of directory with more than 60 000 files.
The classical approach gets an array with all the files and then loop to process the first 10 files.
With the java.nio approach, a stream is opened and the files are processed as needed, the stream is closed after the 10th file.
package misc;
import java.io.File;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
public class TestDir {
public static void main( String[] args ) throws IOException {
int maxFiles = 10;
System.out.println( "TEST BIG DIR" );
nioRun( "\\\\server.local\\files\\20130220", maxFiles );
ioRun( "\\\\server.local\\files\\20130220", maxFiles );
}
// the classical way
private static void ioRun( String filePath, int maxFiles )
throws IOException {
int i = 1;
System.out.println( "IO run" );
long start = System.currentTimeMillis();
File folder = new File( filePath );
File[] listOfFiles = folder.listFiles();
// System.out.println("Total : " + listOfFiles.length);
for (File file : listOfFiles) {
System.out.println( "" + i + ": " + file.getName() );
if (++i > maxFiles) break;
}
long stop = System.currentTimeMillis();
System.out.println( "Elapsed: " + (stop - start) + " ms" );
}
// the new way
private static void nioRun( String filePath, int maxFiles )
throws IOException {
int i = 1;
System.out.println( "NIO run" );
long start = System.currentTimeMillis();
Path dir = FileSystems.getDefault().getPath( filePath );
DirectoryStream<Path> stream = Files.newDirectoryStream( dir );
for (Path path : stream) {
System.out.println( "" + i + ": " + path.getFileName() );
if (++i > maxFiles) break;
}
stream.close();
long stop = System.currentTimeMillis();
System.out.println( "Elapsed: " + (stop - start) + " ms" );
}
}
TEST BIG DIR NIO run 1: 1355970441334_000000_AA000000.PDF 2: 1355970441334_000000_AA000000.props 3: 1355970441335_530025_AA000000.PDF 4: 1355970441335_530025_AA000000.props 5: 1355970441336_832300_AA000000.PDF 6: 1355970441336_832300_AA000000.props 7: 1355970441337_877400_AA000000.PDF 8: 1355970441337_877400_AA000000.props 9: 1355970441338_879900_AA000000.PDF 10: 1355970441338_879900_AA000000.props Elapsed: 158 ms IO run 1: 1355970441334_000000_AA000000.PDF 2: 1355970441334_000000_AA000000.props 3: 1355970441335_530025_AA000000.PDF 4: 1355970441335_530025_AA000000.props 5: 1355970441336_832300_AA000000.PDF 6: 1355970441336_832300_AA000000.props 7: 1355970441337_877400_AA000000.PDF 8: 1355970441337_877400_AA000000.props 9: 1355970441338_879900_AA000000.PDF 10: 1355970441338_879900_AA000000.props Elapsed: 73455 ms