0

我在 nodejs 中使用了一些图像处理库，比如jimp和sharp，我想把它们放在一个工作线程中，因为它们是 cpu 密集型的。然而，在大型图像上使用这些工具时，nodejs 工作线程似乎从不进行垃圾收集。

我已经能够通过对大型源文件的所有操作中最简单的操作来重现该问题，即使用 fs read 将它们读入缓冲区。

下面的代码片段显示，当bigFileTest()在 worker 中执行时，rss 的大小不断增加，但在主线程中执行时会进行垃圾收集。

已尝试以下操作：

  • global.gc()使用With node 选项强制垃圾收集启动--expose-gc没有区别。
  • 为 gc 提供时间来执行它的功能await new Promise(resolve => setTimeout(resolve, 3000));并没有什么区别。
  • 设置最大旧空间的限制没有区别--max-old-space-size=1024
  • 将变量显式设置fileBuffer为 null 没有区别

下面的示例从 wikipedia 获取一个大文件，然后在 service worker 的循环中将其加载到内存中 10 次。每次迭代时，使用的内存都会随着文件大小的增加而增加。

但是，如果执行示例从主线程调用测试，则使用的内存不会增加。

有没有办法在线程内触发垃圾收集。难道我做错了什么？

import path from "path";
import fetch from 'node-fetch';

import {
    Worker,
    isMainThread
} from 'worker_threads'

import {readFile, writeFile, access} from "fs/promises";


const moduleURL = new URL(import.meta.url);
const __dirname = path.dirname(moduleURL.pathname);
const __filename = moduleURL.pathname;


if (isMainThread) {
    await ensureBigFileExists()
    const worker = new Worker(__filename);

    //await bigFileTest()
} else {
    await bigFileTest()
}


async function bigFileTest() {

    let bigFileName = "Google_Art_Project.jpg"
    let imagePath = path.join(__dirname, bigFileName)
    let priorRss = process.memoryUsage().rss
    let baseRss = process.memoryUsage().rss

    let iterations= 10

    for (let iterator = 0; iterator < iterations; iterator++) {
        let fileBuffer = await readFile(imagePath)
        fileBuffer = null

        await new Promise(resolve => setTimeout(resolve, 3000));
        global.gc()

        let diff = Math.round((process.memoryUsage().rss - priorRss) / 1024 / 1024 * 1000) / 1000
        priorRss = process.memoryUsage().rss

        let rssMem = Math.round((process.memoryUsage().rss - baseRss) / 1024 / 1024 * 1000) / 1000
        console.log("iteration:", iterator, "mem:", rssMem, "avg:", Math.round((rssMem) / (iterator+1) * 1000) / 1000, "diff:", diff, JSON.stringify(process.memoryUsage()))
    }

    let diff = Math.round((process.memoryUsage().rss - priorRss) / 1024 / 1024 * 1000) / 1000
    priorRss = process.memoryUsage().rss
    let rssMem = Math.round((process.memoryUsage().rss - baseRss) / 1024 / 1024 * 1000) / 1000
    console.log("out of context:", "mem:", rssMem, "avg:", Math.round((rssMem) / (iterations+1) * 1000) / 1000, "diff:", diff, JSON.stringify(process.memoryUsage()))


}

async function ensureBigFileExists() {
    let bigFileUrl = "https://upload.wikimedia.org/wikipedia/commons/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg"
    let bigFileName = "Google_Art_Project.jpg"
    let imagePath = path.join(__dirname, bigFileName)

    try {
        let fileBuffer = await access(imagePath)
    } catch (error) {
        let fetchResponse = await fetch(bigFileUrl)
        let buffer = await fetchResponse.buffer()
        await writeFile(imagePath, buffer)
    }
}

结果

在工人中执行时

请注意，rss 的大小不断增加，大约是正在加载的文件的大小。但是，heapTotal 和 heapUsed 似乎没有增加。

iteration: 0 mem: 205.254 avg: 205.254 diff: 205.254 {"rss":275357696,"heapTotal":10432512,"heapUsed":4017872,"external":217147950,"arrayBuffers":215130045}
iteration: 1 mem: 410.199 avg: 205.1 diff: 205.078 {"rss":489996288,"heapTotal":7286784,"heapUsed":4014224,"external":432214297,"arrayBuffers":430194808}
iteration: 2 mem: 611.043 avg: 203.681 diff: 201.152 {"rss":700858368,"heapTotal":7548928,"heapUsed":3964488,"external":647279060,"arrayBuffers":645259571}
iteration: 3 mem: 815.637 avg: 203.909 diff: 204.887 {"rss":915390464,"heapTotal":6762496,"heapUsed":3773128,"external":862343823,"arrayBuffers":860324334}
iteration: 4 mem: 1020.715 avg: 204.143 diff: 205.078 {"rss":1130430464,"heapTotal":7286784,"heapUsed":3818600,"external":1077408586,"arrayBuffers":1075407961}
iteration: 5 mem: 1225.738 avg: 204.29 diff: 205.023 {"rss":1345413120,"heapTotal":7286784,"heapUsed":3855704,"external":1292473349,"arrayBuffers":1290464372}
iteration: 6 mem: 1431.215 avg: 204.459 diff: 205.535 {"rss":1560608768,"heapTotal":7548928,"heapUsed":3904560,"external":1507538112,"arrayBuffers":1505518623}
iteration: 7 mem: 1636.289 avg: 204.536 diff: 205.023 {"rss":1775906816,"heapTotal":7548928,"heapUsed":3941224,"external":1722602875,"arrayBuffers":1720591450}
iteration: 8 mem: 1841.816 avg: 204.646 diff: 205.527 {"rss":1991417856,"heapTotal":7811072,"heapUsed":3999456,"external":1937667638,"arrayBuffers":1935680549}
iteration: 9 mem: 2046.406 avg: 204.641 diff: 204.59 {"rss":2205945856,"heapTotal":7548928,"heapUsed":4037768,"external":2152732401,"arrayBuffers":2150743584}
out of context: mem: 2046.098 avg: 186.009 diff: -0.309 {"rss":2205622272,"heapTotal":7548928,"heapUsed":4483000,"external":2152732401,"arrayBuffers":2150712912}

在 main 中执行时

iteration: 0 mem: 208.23 avg: 208.23 diff: 209.145 {"rss":266444800,"heapTotal":11247616,"heapUsed":5534672,"external":216195798,"arrayBuffers":16578}
iteration: 1 mem: 194.699 avg: 97.35 diff: -9.531 {"rss":250159104,"heapTotal":8101888,"heapUsed":5413024,"external":216197670,"arrayBuffers":16578}
iteration: 2 mem: 202.133 avg: 67.378 diff: 11.434 {"rss":260050944,"heapTotal":8101888,"heapUsed":5406288,"external":216197670,"arrayBuffers":16578}
iteration: 3 mem: 198.969 avg: 49.742 diff: -1.164 {"rss":256733184,"heapTotal":8101888,"heapUsed":5261632,"external":216197670,"arrayBuffers":16578}
iteration: 4 mem: 205.285 avg: 41.057 diff: 8.316 {"rss":263356416,"heapTotal":8364032,"heapUsed":5271176,"external":216197670,"arrayBuffers":16578}
iteration: 5 mem: 205.938 avg: 34.323 diff: -1.348 {"rss":261943296,"heapTotal":8364032,"heapUsed":5269520,"external":216197670,"arrayBuffers":16578}
iteration: 6 mem: 204.23 avg: 29.176 diff: 0.293 {"rss":262250496,"heapTotal":8364032,"heapUsed":5277112,"external":216197670,"arrayBuffers":16578}
iteration: 7 mem: 204.934 avg: 25.617 diff: 2.703 {"rss":262987776,"heapTotal":8364032,"heapUsed":5273936,"external":216197670,"arrayBuffers":16578}
iteration: 8 mem: 201.645 avg: 22.405 diff: -1.289 {"rss":259538944,"heapTotal":8364032,"heapUsed":5302160,"external":216197670,"arrayBuffers":16578}
iteration: 9 mem: 205.996 avg: 20.6 diff: 4.352 {"rss":264101888,"heapTotal":8364032,"heapUsed":5303280,"external":216197670,"arrayBuffers":16578}
out of context: mem: 7.234 avg: 0.658 diff: -198.762 {"rss":55685120,"heapTotal":8364032,"heapUsed":5553128,"external":216197670,"arrayBuffers":16578}
4

0 回答 0