我正在开发一个 Java 应用程序来从磁盘读取多个文本文件，修改它们并将修改后的文件写回磁盘。当我使用http://sourceforge.net/projects/filereadtest/files/filereadtest-java-gui/中的“FileReadTestJavaGui-1.05.jar”运行读取测试时，读取速度高达 250 MB/秒。当我运行下面给出的测试应用程序时，我的读取速度低至 11.75 MB/秒。我正在使用 ThreadPoolExecutor 并且无法获得一致的读/写速度。非常欢迎任何有关解决此问题的建议。我在具有 12 GB RAM 的戴尔双四核 2.4 GHz 机器上的 Windows 2008 Server R2 机器上运行了这些测试。
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class RunTest {
private static String baseFileName = "C:/logs/input_";
private static class FileReaderRunnable implements Runnable {
public FileReaderRunnable(int threadID) {
super();
this.threadID = threadID;
}
int threadID = -1;
@Override
public void run() {
Thread.currentThread().setName("Thread-" + threadID);
String writeBaseFileName = "c:/temp/output";
String[] args = {baseFileName + threadID + ".log", writeBaseFileName + threadID + ".txt"};
readAndWriteFileInputOutputStream(args);
}
}
public static void main(String[] args) {
int MAX_THREADS = 20;
int coreThreadPoolSize = 2;
int maxThreadPoolSize = 4;
long threadKeepAliveTime = 12000000;
int arrayBlockQueueSize = 400;
ThreadPoolExecutor executor = new ThreadPoolExecutor(
coreThreadPoolSize,
maxThreadPoolSize,
threadKeepAliveTime,
TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<Runnable>(arrayBlockQueueSize, true));
for (int i = 0; i < MAX_THREADS; i++) {
FileReaderRunnable frr = new FileReaderRunnable(i+1);
executor.execute(frr);
}
executor.shutdown();
while (!executor.isTerminated()) {
try {
Thread.sleep(5000);
print("********** executor.getPoolSize()=" + executor.getPoolSize());
print("********** executor.getCorePoolSize()=" + executor.getCorePoolSize());
print("********** executor.getActiveCount()=" + executor.getActiveCount());
print("********** executor.getCompletedTaskCount()=" + executor.getCompletedTaskCount());
print("********** executor.getTaskCount()=" + executor.getTaskCount());
print("********** executor.isShutdown()=" + executor.isShutdown());
print("********** executor.isTerminated()=" + executor.isTerminated());
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
public static void readAndWriteFileInputOutputStream(String[] args) {
try {
String routineName = Thread.currentThread().getName();
String fInp = "c:\\temp\\input.txt";
String fOut = "c:\\temp\\output.txt";
if( args.length > 0 ) {
fInp = args[0];
}
if( args.length > 1 ) {
fOut = args[1];
}
FileInputStream fis = null;
FileOutputStream fos = null;
// delete the file if it exists
File fff = new File( fOut );
if( fff.exists() ) {
fff.delete();
}
fff = null;
fos = new FileOutputStream(fOut);
fis = new FileInputStream( fInp );
long readTime = 0;
long writeTime = 0;
final int FILE_CHUNK_SIZE = 16 * 1024;
byte[] buffer = new byte[ FILE_CHUNK_SIZE ];
long sTOverall = System.nanoTime();
final long fileLen = fis.available();
for ( long restSize = fileLen; restSize > 0; ) {
final int sizeToProcess = (int) java.lang.Math.min( restSize, FILE_CHUNK_SIZE );
long sT = System.nanoTime();
final int actuallyRead = fis.read( buffer, 0, sizeToProcess );
readTime += System.nanoTime() - sT;
restSize -= actuallyRead;
long sT0 = System.nanoTime();
fos.write(buffer);
writeTime += System.nanoTime() - sT0;
}
sTOverall = System.nanoTime() - sTOverall;
fff = new File(fOut);
double size = ((double)fff.length()) / (1024 * 1024);
double secTaken = ((double)writeTime) / 1000000000;
print(routineName + ": Took: " + secTaken + " sec to write " + size + " MB which is " + (size/secTaken) + " MB/sec");
fos.close();
File fff2 = new File(fInp);
size = ((double)fff2.length()) / (1024 * 1024);
secTaken = ((double)readTime) / 1000000000;
print(routineName + ": Took: " + secTaken + " sec to read " + size + " MB which is " + (size/secTaken) + " MB/sec");
fis.close();
print(routineName + ": Took " + sTOverall + " nanosec overall");
} catch (IOException e) {
e.printStackTrace();
}
}
public static void print(String string) {
System.out.println(string);
}
}
我得到的结果如下：
Thread-1: Took: 0.171561487 sec to write 100.015625 MB which is 582.9724768006936 MB/sec
Thread-2: Took: 0.174120658 sec to write 100.015625 MB which is 574.4041295777781 MB/sec
Thread-1: Took: 0.095193221 sec to read 100.00039100646973 MB which is 1050.4990791988196 MB/sec
Thread-1: Took 273077435 nanosec overall
Thread-2: Took: 0.095017056 sec to read 100.00039100646973 MB which is 1052.4467418404304 MB/sec
Thread-2: Took 273701058 nanosec overall
Thread-4: Took: 0.14674968 sec to write 100.015625 MB which is 681.538964854983 MB/sec
Thread-4: Took: 0.089255036 sec to read 100.00039100646973 MB which is 1120.3893414649424 MB/sec
Thread-4: Took 238318510 nanosec overall
Thread-3: Took: 0.151534826 sec to write 100.015625 MB which is 660.0174206818965 MB/sec
Thread-3: Took: 0.093351422 sec to read 100.00039100646973 MB which is 1071.2251496979845 MB/sec
Thread-3: Took 246756663 nanosec overall
Thread-5: Took: 0.149539881 sec to write 100.015625 MB which is 668.8224193517981 MB/sec
Thread-5: Took: 0.089968802 sec to read 100.00039100646973 MB which is 1111.5007511878364 MB/sec
Thread-5: Took 239927158 nanosec overall
Thread-6: Took: 0.153380444 sec to write 100.015625 MB which is 652.0754692821205 MB/sec
Thread-6: Took: 0.087988256 sec to read 100.00039100646973 MB which is 1136.519753346057 MB/sec
Thread-6: Took 241764966 nanosec overall
Thread-7: Took: 0.142689584 sec to write 100.015625 MB which is 700.9315059745355 MB/sec
Thread-7: Took: 0.088074798 sec to read 100.00039100646973 MB which is 1135.4030128626548 MB/sec
Thread-7: Took 231188741 nanosec overall
Thread-8: Took: 0.140693664 sec to write 100.015625 MB which is 710.8751180152648 MB/sec
Thread-8: Took: 0.085812695 sec to read 100.00039100646973 MB which is 1165.333299536505 MB/sec
Thread-8: Took 226884069 nanosec overall
Thread-9: Took: 0.105057511 sec to write 100.015625 MB which is 952.0083242786895 MB/sec
Thread-9: Took: 0.076369586 sec to read 100.00039100646973 MB which is 1309.4269099019305 MB/sec
Thread-9: Took 181834692 nanosec overall
Thread-10: Took: 0.113160569 sec to write 100.015625 MB which is 883.8381238609713 MB/sec
Thread-10: Took: 0.083640607 sec to read 100.00039100646973 MB which is 1195.5961893780818 MB/sec
Thread-10: Took 197200174 nanosec overall
********** executor.getPoolSize()=2
********** executor.getCorePoolSize()=2
********** executor.getActiveCount()=2
********** executor.getCompletedTaskCount()=10
********** executor.getTaskCount()=20
********** executor.isShutdown()=true
********** executor.isTerminated()=false
Thread-11: Took: 0.154032991 sec to write 100.015625 MB which is 649.3130098343672 MB/sec
Thread-11: Took: 8.420174029 sec to read 100.00039100646973 MB which is 11.876285533061127 MB/sec
Thread-11: Took 8574829737 nanosec overall
********** executor.getPoolSize()=2
********** executor.getCorePoolSize()=2
********** executor.getActiveCount()=2
********** executor.getCompletedTaskCount()=11
********** executor.getTaskCount()=20
********** executor.isShutdown()=true
********** executor.isTerminated()=false
Thread-12: Took: 0.140394781 sec to write 100.015625 MB which is 712.3884825889646 MB/sec
Thread-12: Took: 8.505642503 sec to read 100.00039100646973 MB which is 11.756947340685771 MB/sec
Thread-12: Took 8646511924 nanosec overall
Thread-13: Took: 0.143740254 sec to write 100.015625 MB which is 695.8080441405091 MB/sec
Thread-13: Took: 4.546878135 sec to read 100.00039100646973 MB which is 21.993197978346462 MB/sec
Thread-13: Took 4691105248 nanosec overall
********** executor.getPoolSize()=2
********** executor.getCorePoolSize()=2
********** executor.getActiveCount()=2
********** executor.getCompletedTaskCount()=13
********** executor.getTaskCount()=20
********** executor.isShutdown()=true
********** executor.isTerminated()=false
Thread-14: Took: 0.143977148 sec to write 100.015625 MB which is 694.6631905779936 MB/sec
Thread-14: Took: 5.020245761 sec to read 100.00039100646973 MB which is 19.91942143217911 MB/sec
Thread-14: Took 5164713374 nanosec overall
Thread-15: Took: 0.155099121 sec to write 100.015625 MB which is 644.8497216177002 MB/sec
Thread-15: Took: 3.628340768 sec to read 100.00039100646973 MB which is 27.560914864562612 MB/sec
Thread-15: Took 3783981095 nanosec overall
Thread-16: Took: 0.147386899 sec to write 100.015625 MB which is 678.5923693258518 MB/sec
Thread-16: Took: 3.865473212 sec to read 100.00039100646973 MB which is 25.870154964734425 MB/sec
Thread-16: Took 4013349692 nanosec overall
********** executor.getPoolSize()=2
********** executor.getCorePoolSize()=2
********** executor.getActiveCount()=2
********** executor.getCompletedTaskCount()=16
********** executor.getTaskCount()=20
********** executor.isShutdown()=true
********** executor.isTerminated()=false
Thread-17: Took: 0.145950813 sec to write 100.015625 MB which is 685.2693927782368 MB/sec
Thread-17: Took: 3.594029782 sec to read 100.00039100646973 MB which is 27.824029591324553 MB/sec
Thread-17: Took 3740485565 nanosec overall
Thread-18: Took: 0.143193164 sec to write 100.015625 MB which is 698.4664784696006 MB/sec
Thread-18: Took: 4.387502482 sec to read 100.00039100646973 MB which is 22.792099016861528 MB/sec
Thread-18: Took 4531185662 nanosec overall
********** executor.getPoolSize()=2
********** executor.getCorePoolSize()=2
********** executor.getActiveCount()=2
********** executor.getCompletedTaskCount()=18
********** executor.getTaskCount()=20
********** executor.isShutdown()=true
********** executor.isTerminated()=false
Thread-19: Took: 0.14751012 sec to write 100.015625 MB which is 678.0255144528389 MB/sec
Thread-19: Took: 5.882018447 sec to read 100.00039100646973 MB which is 17.001033218022773 MB/sec
Thread-19: Took 6030072471 nanosec overall
Thread-20: Took: 0.13555387 sec to write 100.015625 MB which is 737.8293589109629 MB/sec
Thread-20: Took: 4.122019495 sec to read 100.00039100646973 MB which is 24.260048048722226 MB/sec
Thread-20: Took 4258033501 nanosec overall
********** executor.getPoolSize()=0
********** executor.getCorePoolSize()=2
********** executor.getActiveCount()=0
********** executor.getCompletedTaskCount()=20
********** executor.getTaskCount()=20
********** executor.isShutdown()=true
********** executor.isTerminated()=true