加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
crawl.js 2.99 KB
一键复制 编辑 原始数据 按行查看 历史
TruthHun 提交于 2019-09-18 21:25 . puppeteer采集优化
'use strict';
const puppeteer = require('puppeteer');
const fs = require("fs");
let args = process.argv.splice(2);
let l=args.length;
let url, folder, selector;
for(let i=0;i<l;i++){
switch (args[i]){
case "--url":
url = args[i+1];
if (url==undefined){
url = "";
}
break;
case '--folder':
folder = args[i+1];
break;
case '--selector':
selector = args.splice(i+1).join(" ");
i=l;
break;
}
i++;
}
async function screenshot() {
const browser = await puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox'], headless: true, ignoreHTTPSErrors: true});
const page = await browser.newPage();
let shot = false;
if(folder && selector){
shot = true;
page.setViewport({width: 1280, height: 20480});
}
page.setExtraHTTPHeaders({
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,co;q=0.7,fr;q=0.6,zh-HK;q=0.5,zh-TW;q=0.4",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3766.2 Safari/537.36"
});
await page.setRequestInterception(true);
page.on("request", request => {
if(getHost(request.url()).indexOf("google")>-1){
request.abort();
}else{
request.continue();
}
});
let timeout = shot ? 120000 : 60000;
await page.goto(url, {"waitUntil" : ['networkidle2', 'domcontentloaded'], "timeout":timeout});
let res;
if(shot){
if (folder.substr(folder.length-1,1)!="/"){
folder=folder+"/"
}
res = await page.evaluate(function (ele) {
let bodyHeight = document.querySelector("body").clientHeight
let data = new Array();
let eleSlice=ele.split(",");
for (let i = 0; i < eleSlice.length; i++) {
let d = [],item = eleSlice[i];
let elements = document.querySelectorAll(item);
for (var element of elements){
let bounding = element.getBoundingClientRect();
let x = bounding.x;
let y = bounding.y;
let width = bounding.width;
let height = bounding.height;
d.push({x, y, width, height});
}
data.push(d)
}
return {height: bodyHeight, data: data};
}, selector);
fs.writeFile(folder+'screenshot.json', JSON.stringify(res),function(){});
await page.screenshot({path: folder+'screenshot.png', fullPage: true});
}
let content=await page.content();
console.log(content);
await browser.close();
}
function getHost(url) {
let u = String(url).toLowerCase()
if (u.startsWith("https://") || u.startsWith("http://")){
return u.split("/")[2]
}
return ""
}
if (url) screenshot();
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化