18

我对使用内存映射 IO 的前景感兴趣，最好利用 boost::interprocess 中的设施来支持跨平台，将文件中的非连续系统页面大小块映射到内存中的连续地址空间。

一个简化的具体场景：

我有许多“普通旧数据”结构，每个结构都是固定长度（小于系统页面大小）。这些结构连接成一个（非常长的）流，结构的类型和位置由在流中处理它们的那些结构的值。我的目标是在要求苛刻的并发环境中最大限度地减少延迟并最大限度地提高吞吐量。

我可以通过将这些数据映射到至少是系统页面大小两倍的块中来非常有效地读取这些数据......并在读取超出倒数第二个系统页面边界的结构后立即建立一个新映射。这允许与普通旧数据结构交互的代码幸福地不知道这些结构是内存映射的……例如，可以直接使用 memcmp() 比较两个不同的结构，而不必关心页面边界。

事情变得有趣的地方在于更新这些数据流......同时（同时）读取它们。我想使用的策略受到系统页面大小粒度上的“写入时复制”的启发......本质上是写入“覆盖页面” - 允许一个进程读取旧数据，而另一个进程读取更新的数据。

虽然管理要使用的覆盖页面以及何时使用并不一定是微不足道的......这不是我主要关心的问题。我主要担心的是我可能有一个跨越第 4 页和第 5 页的结构，然后更新一个完全包含在第 5 页中的结构......在位置 6 中写入新页面......当第 5 页被“垃圾收集”时确定不再可达。这意味着，如果我将第 4 页映射到位置 M，我需要将第 6 页映射到内存位置 M+page_size... 以便能够使用现有的（非内存映射-知道）功能。

我正在尝试制定最佳策略，但我觉得文档不完整，这使我受到了阻碍。本质上，我需要将地址空间的分配从内存映射中解耦到该地址空间。使用 mmap()，我知道我可以使用 MAP_FIXED - 如果我希望明确控制映射位置......但我不清楚我应该如何保留地址空间以安全地执行此操作。我可以在没有 MAP_FIXED 的情况下为两个页面映射 /dev/zero，然后使用 MAP_FIXED 两次将两个页面映射到显式 VM 地址处的分配空间吗？如果是这样，我也应该调用 munmap() 三次吗？它会泄漏资源和/或有任何其他不良开销吗？为了使问题更加复杂，我想在 Windows 上进行类似的行为...... 有什么办法吗？如果我要妥协我的跨平台野心，是否有巧妙的解决方案？

--

感谢您的回答，Mahmoud ...我已经阅读并认为我已经理解了该代码...我已经在 Linux 下编译了它，它的行为符合您的建议。

我主要关心的是第 62 行 - 使用 MAP_FIXED。它对 mmap 做了一些假设，当我阅读我能找到的文档时，我无法确认这些假设。您将“更新”页面映射到与最初返回的 mmap() 相同的地址空间 - 我认为这是“正确的” - 即不是碰巧在 Linux 上工作的东西？我还需要假设它适用于文件映射和匿名映射的跨平台。

该示例确实使我前进...记录了我最终需要的内容可能可以通过 Linux 上的 mmap() 来实现——至少。我真正想要的是一个指向文档的指针，该文档显示 MAP_FIXED 行将像示例演示的那样工作......并且，理想情况下，从 Linux/Unix 特定 mmap() 到独立于平台的转换（Boost::interprocess ） 方法。

4

3 回答 3

7

你的问题有点令人困惑。据我了解，此代码将满足您的需求：

#define PAGESIZE 4096

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <errno.h>
#include <sys/types.h>
#include <fcntl.h>
#include <unistd.h>
#include <assert.h>

struct StoredObject
{
    int IntVal;
    char StrVal[25];
};

int main(int argc, char **argv)
{
    int fd = open("mmapfile", O_RDWR | O_CREAT | O_TRUNC, (mode_t) 0600);
    //Set the file to the size of our data (2 pages)
    lseek(fd, PAGESIZE*2 - 1, SEEK_SET);
    write(fd, "", 1); //The final byte

    unsigned char *mapPtr = (unsigned char *) mmap(0, PAGESIZE * 2, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);

    struct StoredObject controlObject;
    controlObject.IntVal = 12;
    strcpy(controlObject.StrVal, "Mary had a little lamb.\n");

    struct StoredObject *mary1;
    mary1 = (struct StoredObject *)(mapPtr + PAGESIZE - 4); //Will fall on the boundary between first and second page
    memcpy(mary1, &controlObject, sizeof(StoredObject));

    printf("%d, %s", mary1->IntVal, mary1->StrVal);
    //Should print "12, Mary had a little lamb.\n"

    struct StoredObject *john1;
    john1 = mary1 + 1; //Comes immediately after mary1 in memory; will start and end in the second page
    memcpy(john1, &controlObject, sizeof(StoredObject));

    john1->IntVal = 42;
    strcpy(john1->StrVal, "John had a little lamb.\n");

    printf("%d, %s", john1->IntVal, john1->StrVal);
    //Should print "12, Mary had a little lamb.\n"

    //Make sure the data's on the disk, as this is the initial, "read-only" data
    msync(mapPtr, PAGESIZE * 2, MS_SYNC);

    //This is the inital data set, now in memory, loaded across two pages
    //At this point, someone could be reading from there. We don't know or care.
    //We want to modify john1, but don't want to write over the existing data
    //Easy as pie.

    //This is the shadow map. COW-like optimization will take place: 
    //we'll map the entire address space from the shared source, then overlap with a new map to modify
    //This is mapped anywhere, letting the system decide what address we'll be using for the new data pointer
    unsigned char *mapPtr2 = (unsigned char *) mmap(0, PAGESIZE * 2, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);

    //Map the second page on top of the first mapping; this is the one that we're modifying. It is *not* backed by disk
    unsigned char *temp = (unsigned char *) mmap(mapPtr2 + PAGESIZE, PAGESIZE, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED | MAP_ANON, 0, 0);
    if (temp == MAP_FAILED)
    {
        printf("Fixed map failed. %s", strerror(errno));
    }
    assert(temp == mapPtr2 + PAGESIZE);

    //Make a copy of the old data that will later be changed
    memcpy(mapPtr2 + PAGESIZE, mapPtr + PAGESIZE, PAGESIZE);

    //The two address spaces should still be identical until this point
    assert(memcmp(mapPtr, mapPtr2, PAGESIZE * 2) == 0);

    //We can now make our changes to the second page as needed
    struct StoredObject *mary2 = (struct StoredObject *)(((unsigned char *)mary1 - mapPtr) + mapPtr2);
    struct StoredObject *john2 = (struct StoredObject *)(((unsigned char *)john1 - mapPtr) + mapPtr2);

    john2->IntVal = 52;
    strcpy(john2->StrVal, "Mike had a little lamb.\n");

    //Test that everything worked OK
    assert(memcmp(mary1, mary2, sizeof(struct StoredObject)) == 0);
    printf("%d, %s", john2->IntVal, john2->StrVal);
    //Should print "52, Mike had a little lamb.\n"

    //Now assume our garbage collection routine has detected that no one is using the original copy of the data
    munmap(mapPtr, PAGESIZE * 2);

    mapPtr = mapPtr2;

    //Now we're done with all our work and want to completely clean up
    munmap(mapPtr2, PAGESIZE * 2);

    close(fd);

    return 0;
}

我修改后的答案应该可以解决您的安全问题。仅MAP_FIXED在第二次mmap通话时使用（就像我上面所说的那样）。很酷的一点MAP_FIXED是，它可以让您覆盖现有的mmap地址部分。它将卸载您重叠的范围并将其替换为您的新映射内容：

 MAP_FIXED
              [...] If the memory
              region specified by addr and len overlaps pages of any existing
              mapping(s), then the overlapped part of the existing mapping(s) will be
              discarded. [...]

这样，您就可以让操作系统为您找到数百兆的连续内存块（永远不要调用MAP_FIXED您不确定的地址不可用）。然后你MAP_FIXED用你将要修改的数据调用现在映射的巨大空间的一个子部分。多田。

在 Windows 上，这样的东西应该可以工作（我现在在 Mac 上，所以未经测试）：

int main(int argc, char **argv)
{
    HANDLE hFile = CreateFile(L"mmapfile", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
    //Set the file to the size of our data (2 pages)
    SetFilePointer(hFile, PAGESIZE*2 - 1, 0, FILE_BEGIN);
    DWORD bytesWritten = -1;
    WriteFile(hFile, "", 1, &bytesWritten, NULL);

    HANDLE hMap = CreateFileMapping(hFile, NULL, PAGE_READWRITE, 0, PAGESIZE * 2, NULL);
    unsigned char *mapPtr = (unsigned char *) MapViewOfFile(hMap, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, PAGESIZE * 2);

    struct StoredObject controlObject;
    controlObject.IntVal = 12;
    strcpy(controlObject.StrVal, "Mary had a little lamb.\n");

    struct StoredObject *mary1;
    mary1 = (struct StoredObject *)(mapPtr + PAGESIZE - 4); //Will fall on the boundary between first and second page
    memcpy(mary1, &controlObject, sizeof(StoredObject));

    printf("%d, %s", mary1->IntVal, mary1->StrVal);
    //Should print "12, Mary had a little lamb.\n"

    struct StoredObject *john1;
    john1 = mary1 + 1; //Comes immediately after mary1 in memory; will start and end in the second page
    memcpy(john1, &controlObject, sizeof(StoredObject));

    john1->IntVal = 42;
    strcpy(john1->StrVal, "John had a little lamb.\n");

    printf("%d, %s", john1->IntVal, john1->StrVal);
    //Should print "12, Mary had a little lamb.\n"

    //Make sure the data's on the disk, as this is the initial, "read-only" data
    //msync(mapPtr, PAGESIZE * 2, MS_SYNC);

    //This is the inital data set, now in memory, loaded across two pages
    //At this point, someone could be reading from there. We don't know or care.
    //We want to modify john1, but don't want to write over the existing data
    //Easy as pie.

    //This is the shadow map. COW-like optimization will take place: 
    //we'll map the entire address space from the shared source, then overlap with a new map to modify
    //This is mapped anywhere, letting the system decide what address we'll be using for the new data pointer
    unsigned char *reservedMem = (unsigned char *) VirtualAlloc(NULL, PAGESIZE * 2, MEM_RESERVE, PAGE_READWRITE);
    HANDLE hMap2 = CreateFileMapping(hFile, NULL, PAGE_READWRITE, 0, PAGESIZE, NULL);
    unsigned char *mapPtr2 = (unsigned char *) MapViewOfFileEx(hMap2, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, PAGESIZE, reservedMem);

    //Map the second page on top of the first mapping; this is the one that we're modifying. It is *not* backed by disk
    unsigned char *temp = (unsigned char *) MapViewOfFileEx(hMap2, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, PAGESIZE, reservedMem + PAGESIZE);
    if (temp == NULL)
    {
        printf("Fixed map failed. 0x%x\n", GetLastError());
        return -1;
    }
    assert(temp == mapPtr2 + PAGESIZE);

    //Make a copy of the old data that will later be changed
    memcpy(mapPtr2 + PAGESIZE, mapPtr + PAGESIZE, PAGESIZE);

    //The two address spaces should still be identical until this point
    assert(memcmp(mapPtr, mapPtr2, PAGESIZE * 2) == 0);

    //We can now make our changes to the second page as needed
    struct StoredObject *mary2 = (struct StoredObject *)(((unsigned char *)mary1 - mapPtr) + mapPtr2);
    struct StoredObject *john2 = (struct StoredObject *)(((unsigned char *)john1 - mapPtr) + mapPtr2);

    john2->IntVal = 52;
    strcpy(john2->StrVal, "Mike had a little lamb.\n");

    //Test that everything worked OK
    assert(memcmp(mary1, mary2, sizeof(struct StoredObject)) == 0);
    printf("%d, %s", john2->IntVal, john2->StrVal);
    //Should print "52, Mike had a little lamb.\n"

    //Now assume our garbage collection routine has detected that no one is using the original copy of the data
    //munmap(mapPtr, PAGESIZE * 2);

    mapPtr = mapPtr2;

    //Now we're done with all our work and want to completely clean up
    //munmap(mapPtr2, PAGESIZE * 2);

    //close(fd);

    return 0;
}
于 2012-05-07T02:48:11.143 回答
3

但我不清楚我应该如何保留地址空间以便安全地做到这一点

这会因操作系统而异，但是稍微挖掘 msdn 上的 mmap（我从 msdn 搜索中的“xp mmap”开始）显示 Microsoft 有他们通常的 VerboseAndHelpfullyCapitalizedNames 用于实现 mmap 片段的（许多）函数。文件映射器和匿名映射器都可以像任何 POSIX-2001 系统一样处理固定地址请求，即如果地址空间中的其他东西正在与内核通信，您可以对其进行排序。我绝不会“安全地”接触，对于您想要移植到未指定平台的代码，没有“安全地”之类的东西。您将不得不建立自己的预映射匿名内存池，以后您可以在自己的控制下取消映射和打包。

于 2012-05-13T04:55:51.023 回答
1

我从@Mahmoud 测试了windows 代码，实际上我测试了以下类似的代码，但它不起作用（Linux 代码有效。）如果你取消注释VirtualFree，它会起作用。正如我在上面的评论中提到的，在 Windows 上，您可以使用 VirtualAlloc 保留地址空间，但您不能将 MapViewOfFileEx 与已经映射的地址一起使用，因此您需要先 VirtualFree 它。然后有一个竞争条件，另一个线程可以在你之前抢占内存地址，所以你必须在一个循环中做所有事情，例如尝试最多 1000 次然后放弃。

package main

import (
    "fmt"
    "os"
    "syscall"
)

func main() {
    const size = 1024 * 1024

    file, err := os.Create("foo.dat")
    if err != nil {
        panic(err)
    }

    if err := file.Truncate(size); err != nil {
        panic(err)
    }

    const MEM_COMMIT = 0x1000

    addr, err := virtualAlloc(0, size, MEM_COMMIT, protReadWrite)
    if err != nil {
        panic(err)
    }

    fd, err := syscall.CreateFileMapping(
        syscall.Handle(file.Fd()),
        nil,
        uint32(protReadWrite),
        0,
        uint32(size),
        nil,
    )

    //if err := virtualFree(addr); err != nil {
    //  panic(err)
    //}

    base, err := mapViewOfFileEx(fd, syscall.FILE_MAP_READ|syscall.FILE_MAP_WRITE, 0, 0, size, addr)
    if base == 0 {
        panic("mapViewOfFileEx returned 0")
    }
    if err != nil {
        panic(err)
    }

    fmt.Println("success!")
}

type memProtect uint32

const (
    protReadOnly  memProtect = 0x02
    protReadWrite memProtect = 0x04
    protExecute   memProtect = 0x20
    protAll       memProtect = 0x40
)

var (
    modkernel32         = syscall.MustLoadDLL("kernel32.dll")
    procMapViewOfFileEx = modkernel32.MustFindProc("MapViewOfFileEx")
    procVirtualAlloc    = modkernel32.MustFindProc("VirtualAlloc")
    procVirtualFree     = modkernel32.MustFindProc("VirtualFree")
    procVirtualProtect  = modkernel32.MustFindProc("VirtualProtect")
)

func mapViewOfFileEx(handle syscall.Handle, prot memProtect, offsetHigh uint32, offsetLow uint32, length uintptr, target uintptr) (addr uintptr, err error) {
    r0, _, e1 := syscall.Syscall6(procMapViewOfFileEx.Addr(), 6, uintptr(handle), uintptr(prot), uintptr(offsetHigh), uintptr(offsetLow), length, target)
    addr = uintptr(r0)
    if addr == 0 {
        if e1 != 0 {
            err = error(e1)
        } else {
            err = syscall.EINVAL
        }
    }
    return addr, nil
}

func virtualAlloc(addr, size uintptr, allocType uint32, prot memProtect) (mem uintptr, err error) {
    r0, _, e1 := syscall.Syscall6(procVirtualAlloc.Addr(), 4, addr, size, uintptr(allocType), uintptr(prot), 0, 0)
    mem = uintptr(r0)
    if e1 != 0 {
        return 0, error(e1)
    }
    return mem, nil
}

func virtualFree(addr uintptr) error {
    const MEM_RELEASE = 0x8000
    _, _, e1 := syscall.Syscall(procVirtualFree.Addr(), 3, addr, 0, MEM_RELEASE)
    if e1 != 0 {
        return error(e1)
    }
    return nil
}
于 2015-07-14T13:41:14.163 回答