所以我找到了一个有很酷图片的网站,我想抓取它的一些数据。该网站大约 5 年没有得到任何更新,我试图联系它的所有者以获取某种 API,但我没有得到任何回复。
无论如何,网站有分类,每张图片都有自己的页码;所以为了抓取每张图片,我需要转到每个类别,然后转到该特定类别的每个页面。
下面是我的代码,但我无法for loop
重置。
const {Cluster} = require('puppeteer-cluster');
const puppeteer = require('puppeteer');
let c = 0;
let z = 500;
(async () => {
process.setMaxListeners(5);
const cluster = await Cluster.launch({
maxConcurrency: 3 // max browsers to spawn at the same time
});
let b = 20;
for (let i = 0; i < b; i++) {
cluster.execute({i}, async () => {
let browser = await puppeteer.launch({headless: false});
// scraping code using the i and c values
await browser.close();
console.log(i);
if (i > b - 10) {
i = 0;
c = c + 1;
console.log('c = ' + c);
if (c > z)
process.exit();
}
});
}
await cluster.idle();
await cluster.close();
})();
这是输出(顺序不是必需的):
1
0
2
4
3
5
6
7
8
9
10
11
c = 1
12
c = 2
13
c = 3
14
c = 4
16
c = 5
15
c = 6
17
c = 7
18
c = 8
19
c = 9
Process finished with exit code 0
如果我await
在前面添加cluster.execute
然后for loop
正在重置,但是我不能同时使用多个浏览器。
编辑:
const {Cluster} = require('puppeteer-cluster');
const puppeteer = require('puppeteer');
(async () => {
process.setMaxListeners(5);
const cluster = await Cluster.launch({maxConcurrency: 3});
let b = 15;
let d;
function myLoop() {
let g = 0;
for (g; g <= n; g++) {
console.log(g);
myFunc();
}
return g;
}
d = myLoop();
console.log('d: ' + d);
if (d > 0)
myLoop();
async function myFunc() {
await cluster.execute(async () => {
let browser = await puppeteer.launch({headless: false});
await browser.close();
});
}
await cluster.idle();
await cluster.close();
})();