a
/
bifm
2
2
Fork 4
Code Issues Pull Requests Projects Releases Wiki Activity

fix 1bitspace, improve gplinks, fix older node issues

Browse Source
main
aria 10 months ago
parent a988476cb5
commit 80c6061215
Signed by: a
GPG Key ID: E851AE999FFCBC37
17 changed files with 58 additions and 20 deletions
Show Stats Download Patch File Download Diff File

3
docs/dev-docs/CONTRIBUTING-EXT/WRITING-EXTRACTORS.md
Unescape Escape View File

@ -65,6 +65,7 @@ Copy and paste this template into your extractor.
const pw = require("playwright-extra");
//const stl = require("puppeteer-extra-plugin-stealth");
//const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
//const got = require("got");
//const cap = require("@extra/recaptcha");
const lib = require("../lib");
@ -84,7 +85,7 @@ module.exports = {
if (lib.config.debug == true) console.log("[scraper] Launching browser...");
b = await pup.launch({headless: true});
b = await pw.firefox.launch({headless: true});
let p = await b.newPage();
/*

8
extractors/1bitspace.js
Unescape Escape View File

@ -1,5 +1,6 @@
const pw = require("playwright-extra");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const fetch = require("cross-fetch");
const stl = require("puppeteer-extra-plugin-stealth");
const lib = require("../lib");
@ -20,11 +21,16 @@ module.exports = {
}
if (lib.config.debug == true) console.log("[1bitspace] Launching browser...");
let blocker = await PlaywrightBlocker.fromPrebuiltFull(fetch);
let a = (lib.config.defaults?.puppeteer || {headless: true});
b = await pup.launch(a);
b = await pw.firefox.launch(a);
p = await b.newPage();
await blocker.enableBlockingInPage(p);
if (opt.referer) {
if (lib.config.debug == true) console.log("[1bitspace] Going to referer URL first...");
await p.goto(opt.referer, {waitUntil: "domcontentloaded"});

7
extractors/adlinkfly.js
Unescape Escape View File

@ -1,6 +1,7 @@
const pw = require("playwright-extra");
const stl = require("puppeteer-extra-plugin-stealth");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const fetch = require("cross-fetch");
const lib = require("../lib");
module.exports = {
@ -15,13 +16,11 @@ module.exports = {
if (lib.config.fastforward == true && opt.ignoreFF !== true) {
let r = await lib.fastforward.get(url, true);
if (r !== null) {
return {destination: r, fastforward: true};
}
if (r !== null) return {destination: r, fastforward: true};
}
let hn = new URL(url).hostname;
let blocker = await PlaywrightBlocker.fromPrebuiltFull();
let blocker = await PlaywrightBlocker.fromPrebuiltFull(fetch);
if (lib.config.captcha.active == false) {
throw "Captcha service is required for this link, but this instance doesn't support it."

1
extractors/bcvc.js
Unescape Escape View File

@ -30,6 +30,7 @@ module.exports = {
if (lib.config.debug == true) console.log("[bcvc] Launched. Counting down...");
await p.waitForSelector("#getLink", {visible: true});
await p.evaluate(function () {if (document.querySelector(".bvmipp-m-alert")) document.querySelector(".bvmipp-m-alert").remove()})
await p.click("#getLink");
await p.waitForNavigation();

5
extractors/cutwin.js
Unescape Escape View File

@ -1,5 +1,6 @@
const pw = require("playwright-extra");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const fetch = require("cross-fetch");
const stl = require("puppeteer-extra-plugin-stealth");
const lib = require("../lib");
@ -8,7 +9,7 @@ module.exports = {
get: async function(url, opt) {
let b;
try {
let blocker = await PlaywrightBlocker.fromPrebuiltFull();
let blocker = await PlaywrightBlocker.fromPrebuiltFull(fetch);
let stlh = stl();
stlh.enabledEvasions.delete("user-agent-override");
pw.firefox.use(stlh);
@ -16,8 +17,6 @@ module.exports = {
if (lib.config.captcha.active == false) {
throw "Captcha service is required for this link, but this instance doesn't support it."
}
if (lib.config.debug == true) console.log("[cutwin] Launching browser...");
let args = (lib.config.defaults?.puppeteer || {headless: true});

3
extractors/exeio.js
Unescape Escape View File

@ -1,5 +1,6 @@
const pw = require("playwright-extra");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const fetch = require("cross-fetch");
const stl = require("puppeteer-extra-plugin-stealth");
const lib = require("../lib")
@ -9,7 +10,7 @@ module.exports = {
get: async function(url, opt) {
let b;
try {
let blocker = await PlaywrightBlocker.fromPrebuiltFull();
let blocker = await PlaywrightBlocker.fromPrebuiltFull(fetch);
let stlh = stl();
stlh.enabledEvasions.delete("user-agent-override");
pw.firefox.use(stlh);

1
extractors/ez4.js
Unescape Escape View File

@ -1,5 +1,6 @@
const pw = require("playwright-extra");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const fetch = require("cross-fetch");
const stl = require("puppeteer-extra-plugin-stealth");
const lib = require("../lib");

12
extractors/gplinks.js
Unescape Escape View File

@ -1,5 +1,6 @@
const pw = require("playwright-extra");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const fetch = require("cross-fetch");
const stl = require("puppeteer-extra-plugin-stealth");
const lib = require("../lib");
@ -10,7 +11,7 @@ module.exports = {
let b;
try {
if (lib.config.debug == true) console.log("[gplinks] Launching browser...");
let blocker = await PlaywrightBlocker.fromPrebuiltFull();
let blocker = await PlaywrightBlocker.fromPrebuiltFull(fetch);
/*
let stlh = stl();
@ -30,8 +31,8 @@ module.exports = {
if (lib.config.debug == true) console.log("[gplinks] Going to referer URL first...");
await p.goto(opt.referer, {waitUntil: "domcontentloaded"});
}
await p.goto(url, {waitUntil: "load", timeout: 60000});
await p.goto(url, {waitUntil: "domcontentloaded"});
if (lib.config.debug == true) console.log("[gplinks] Launched. Counting down...");
await p.evaluate(function() {
@ -40,7 +41,7 @@ module.exports = {
}, 100);
});
await p.waitForSelector("#btn6", {timeout: (1000 * 45)});
await p.waitForSelector("#btn6", {timeout: 999999});
if (lib.config.debug == true) console.log("[gplinks] Done. Going to next page...");
await p.evaluate(function() {
if (document.body.classList.contains("modal-open")) {
@ -49,13 +50,14 @@ module.exports = {
document.body.classList.remove("modal-open");
}
});
await p.waitForTimeout(2000);
await p.click("#btn6");
await p.waitForSelector("#captchaShortlink > div > div > iframe");
await p.waitForTimeout(1000);
await blocker.enableBlockingInPage(p);
if (lib.config.debug == true) console.log("[gplinks] Done. Solving CAPTCHA...");
await lib.solveThroughPage(p);
await blocker.enableBlockingInPage(p);
await p.evaluate(function() {
document.forms[0].submit();
});

1
extractors/keeplinks.js
Unescape Escape View File

@ -1,6 +1,7 @@
const pw = require("playwright-extra");
const stl = require("puppeteer-extra-plugin-stealth");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const fetch = require("cross-fetch");
const lib = require("../lib");
module.exports = {

3
extractors/link1s.js
Unescape Escape View File

@ -1,6 +1,7 @@
const pw = require("playwright-extra");
const stl = require("puppeteer-extra-plugin-stealth");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const fetch = require("cross-fetch");
const lib = require("../lib");
module.exports = {
@ -13,7 +14,7 @@ module.exports = {
stlh.enabledEvasions.delete("user-agent-override");
pw.firefox.use(stlh);
let blocker = await PlaywrightBlocker.fromPrebuiltFull();
let blocker = await PlaywrightBlocker.fromPrebuiltFull(fetch);
if (lib.config.debug == true) console.log("[link1s] Launching browser...");
let args = (lib.config.defaults?.puppeteer || {headless: true});

3
extractors/lnk2.js
Unescape Escape View File

@ -1,5 +1,6 @@
const pw = require("playwright-extra");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const fetch = require("cross-fetch");
const stl = require("puppeteer-extra-plugin-stealth");
const lib = require("../lib");
@ -15,7 +16,7 @@ module.exports = {
stlh.enabledEvasions.delete("user-agent-override");
pw.firefox.use(stlh);
let blocker = await PlaywrightBlocker.fromPrebuiltFull();
let blocker = await PlaywrightBlocker.fromPrebuiltFull(fetch);
if (lib.config.captcha.active == false) {
throw "Captcha service is required for this link, but this instance doesn't support it."

1
extractors/longwpsafe.js
Unescape Escape View File

@ -1,5 +1,4 @@
const pw = require("playwright-extra");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const stl = require("puppeteer-extra-plugin-stealth");
const lib = require("../lib");

3
extractors/mylink.js
Unescape Escape View File

@ -1,5 +1,6 @@
const pw = require("playwright-extra");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const fetch = require("cross-fetch");
const stl = require("puppeteer-extra-plugin-stealth");
const lib = require("../lib");
@ -15,7 +16,7 @@ module.exports = {
// setting up plugins
let blocker = await PlaywrightBlocker.fromPrebuiltFull();
let blocker = await PlaywrightBlocker.fromPrebuiltFull(fetch);
let stlh = stl();
stlh.enabledEvasions.delete("user-agent-override");
pw.firefox.use(stlh);

1
extractors/try2link.js
Unescape Escape View File

@ -1,5 +1,4 @@
const pw = require("playwright-extra");
const { PlaywrightBlocker } = require("@cliqz/adblocker-playwright");
const lib = require("../lib");
module.exports = {

8
lib.js
Unescape Escape View File

@ -481,6 +481,14 @@ async function solveThroughPage(p) {
if (config.debug == true) console.log("[captcha] Retrieved. Solving CAPTCHA...");
let c = await solveCaptcha(sk, type, {referer: (await p.url())});
if (type == "hcaptcha") {
let id = await p.evaluate(function() {return document.querySelector("iframe[title='widget containing checkbox for hCaptcha security challenge']").src.split("id=")[1].split("&")[0];});
let json = JSON.stringify({id: id, label: "challenge-closed", source: "hcaptcha", contents: {event: 'challenge-passed', expiration: 120, response: c}});
await p.evaluate(`window.postMessage('${json}', '*');`);
}
if (config.debug == true) console.log("[captcha] Solved CAPTCHA. Enterring solution and submitting form...");
await p.evaluate(`document.querySelector("[name='g-recaptcha-response']").value = "${c}";`);
if (type == "hcaptcha") await p.evaluate(`document.querySelector("[name='h-captcha-response']").value = "${c}";`);

17
package-lock.json generated
Unescape Escape View File

@ -12,6 +12,7 @@
"@cliqz/adblocker-playwright": "^1.23.8",
"2captcha": "^3.0.5",
"cheerio": "^1.0.0-rc.12",
"cross-fetch": "^3.1.5",
"ejs": "^3.1.8",
"express": "^4.18.1",
"got": "^11.8.5",
@ -590,6 +591,14 @@
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
"integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
},
"node_modules/cross-fetch": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
"integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==",
"dependencies": {
"node-fetch": "2.6.7"
}
},
"node_modules/css-select": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
@ -2677,6 +2686,14 @@
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
"integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
},
"cross-fetch": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
"integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==",
"requires": {
"node-fetch": "2.6.7"
}
},
"css-select": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",

1
package.json
Unescape Escape View File

@ -18,6 +18,7 @@
"@cliqz/adblocker-playwright": "^1.23.8",
"2captcha": "^3.0.5",
"cheerio": "^1.0.0-rc.12",
"cross-fetch": "^3.1.5",
"ejs": "^3.1.8",
"express": "^4.18.1",
"got": "^11.8.5",

Write Preview
Loading…
Cancel
Save