1

我已经使用两个数组构建了一个自定义哈希图。一个包含键另一个值。现在，我看到有时 JVM 无法为这些数组分配所需的内存并抛出异常。有没有办法使用页面交换或任何其他方法来解决这个问题？

代码：

public class Test1{
public static void main(String args[]){


    try {

   int arrays[]  = new int[50000000] ;
for ( int i = 0; i < 50000000 ; i++)
        {
        arrays[i] = i ;
        }

    }

catch(Exception e){
        System.out.println(e) ;
    }
}
}

编辑：

    public void load() {
        try {
            FileChannel channel2 = new RandomAccessFile(str1, "r").getChannel();
            MappedByteBuffer mbb2 = channel2.map(FileChannel.MapMode.READ_ONLY, 0, channel2.size());
            mbb2.order(ByteOrder.nativeOrder());
            assert mbb2.remaining() == savenum * 8;
            for (int i = 0; i < savenum; i++) {
                long l = mbb2.getLong();
                keys[i] = l;
            }
            channel2.close();

            FileChannel channel3 = new RandomAccessFile(str2, "r").getChannel();
            MappedByteBuffer mbb3 = channel3.map(FileChannel.MapMode.READ_ONLY, 0, channel3.size());
            mbb3.order(ByteOrder.nativeOrder());
            assert mbb3.remaining() == savenum * 4;
            for (int i = 0; i < savenum; i++) {
                int l1 = mbb3.getInt();
                values[i] = l1;
            }
            channel3.close();
        } catch (Exception e) {
            System.out.println(e);
        }
    }

    public void save() {
        try {
            FileChannel channel = new RandomAccessFile(str1, "rw").getChannel();
            MappedByteBuffer mbb = channel.map(FileChannel.MapMode.READ_WRITE, 0, savenum * 8);
            mbb.order(ByteOrder.nativeOrder());

            for (int i = 0; i < savenum; i++) {
                mbb.putLong(keys[i]);
            }
            channel.close();

            FileChannel channel1 = new RandomAccessFile(str2, "rw").getChannel();
            MappedByteBuffer mbb1 = channel1.map(FileChannel.MapMode.READ_WRITE, 0, savenum * 4);
            mbb1.order(ByteOrder.nativeOrder());

            for (int i = 0; i < savenum; i++) {
                mbb1.putInt(values[i]);
            }
            channel1.close();
        } catch (Exception e) {
            System.out.println("IOException : " + e);
        }
    }
4

1 回答 1

1

修改代码以使用 IntBuffer 和 LongBuffer

class LongIntParallelHashMultimap {
    private static final int NULL = 0;
    private final FileChannel channel1, channel2;
    private final LongBuffer keys;
    private final IntBuffer values;
    private final int capacity;
    private int size;

    public LongIntParallelHashMultimap(int capacity, String basename) throws IOException {
        assert (capacity & (capacity - 1)) == 0 : "Capacity " + capacity + " must be a power of 2";
        this.capacity = capacity;
        channel1 = new RandomAccessFile(basename + ".keys", "rw").getChannel();
        keys = channel1.map(FileChannel.MapMode.READ_WRITE, 0, capacity * 8).order(ByteOrder.nativeOrder()).asLongBuffer();
        // load keys into memory
        for(int i=0;i<capacity;i+=512) keys.get(i);

        channel2 = new RandomAccessFile(basename + ".values", "rw").getChannel();
        values = channel2.map(FileChannel.MapMode.READ_WRITE, 0, capacity * 4).order(ByteOrder.nativeOrder()).asIntBuffer();
        for(int i=0;i<capacity;i+=1024) values.get(i);
    }

    public void put(long key, int value) {
        long key1 = key + 1;
        int index = indexFor(key1);
        while (keys.get(index) != NULL) {
            index = successor(index);
        }
        values.put(index, value);
        keys.put(index, key1);
        ++size;
    }

    /**
     * Uses a pre-allocated array and return the count of matches.
     */
    public int get(long key, int[] hits) {
        long key1 = key + 1;
        int index = indexFor(key1);
        int hitIndex = 0;

        while (keys.get(index) != NULL) {
            if (keys.get(index) == key1) {
                hits[hitIndex] = values.get(index);
                ++hitIndex;
            }
            index = successor(index);
        }

        return hitIndex;
    }

    private int indexFor(long key) {
        return Math.abs((int) ((key * 5700357409661598721L) & (capacity - 1)));
    }

    private int successor(int index) {
        return (index + 1) & (capacity - 1);
    }

    public int size() {
        return size;
    }

    public void close() {
        try {
            channel1.close();
            channel2.close();
        } catch (IOException ignored) {
        }
        try {
            ((DirectBuffer) keys).cleaner().clean();
            ((DirectBuffer) values).cleaner().clean();
        } catch (Throwable notSupportedOnThisPlatform) {
        }
    }
}

要允许操作系统交换数据结构的页面，您需要使用堆外内存或内存映射文件。内存映射文件更易于管理，并且可以达到硬盘驱动器的大小。

long heap = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
RandomAccessFile raf = new RandomAccessFile("array.dat", "rw");
IntBuffer map = raf.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, 1 << 30).order(ByteOrder.nativeOrder()).asIntBuffer();
for (int i = 0; i < map.capacity(); i++)
    map.put(i, i);
long heap2 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
System.out.printf("Wrote %,d int values, heap used %,d bytes approx%n", map.capacity(), heap2 - heap);

印刷

Wrote 268,435,456 int values, heap used 0 approx

顺便说一句：您需要一个 64 位 JVM 来一次映射这些数据。如果您有 32 位 JVM，则需要移动映射（这很痛苦，因此请尽可能使用 64 位 JVM）

如果您有一个 200 MB 的数组，它应该适合 32 位 JVM。

于 2012-07-03T19:22:26.443 回答