|
|
|
@ -59,432 +59,445 @@ let client, db, links; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
module.exports = { |
|
|
|
|
get: async function(url, opt) { |
|
|
|
|
try { |
|
|
|
|
if (new URL(url).host.substring(new URL(url).host.length - 8) == "carrd.co") url = url.split("#")[0]; |
|
|
|
|
get: get,
|
|
|
|
|
solve: solveCaptcha, |
|
|
|
|
solveThroughPage: solveThroughPage, |
|
|
|
|
cookieString: cookieString, |
|
|
|
|
fastforward: { |
|
|
|
|
get: ffGet, |
|
|
|
|
send: ffSend |
|
|
|
|
}, |
|
|
|
|
byteCount: byteCount, |
|
|
|
|
config: config, |
|
|
|
|
isUrl: isUrl, |
|
|
|
|
db: { |
|
|
|
|
get: dbGet |
|
|
|
|
}, |
|
|
|
|
cloudflare: { |
|
|
|
|
email: cfEmail |
|
|
|
|
}, |
|
|
|
|
cacheCount: cacheCount, |
|
|
|
|
fixSubdomain: fixSubdomain |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
let extractor = await ext.fromUrl(url); |
|
|
|
|
extractor = require(`./extractors/${extractor}`); |
|
|
|
|
async function get(url, opt) { |
|
|
|
|
try { |
|
|
|
|
if (new URL(url).host.substring(new URL(url).host.length - 8) == "carrd.co") url = url.split("#")[0]; |
|
|
|
|
|
|
|
|
|
if (opt.ignoreCache) opt.ignoreCache = parseBool(opt.ignoreCache); |
|
|
|
|
if (opt.allowCache) opt.allowCache = parseBool(opt.allowCache); |
|
|
|
|
if (opt.ignoreFF) opt.ignoreFF = parseBool(opt.ignoreFF); |
|
|
|
|
if (opt.allowFF) opt.allowFF = parseBool(opt.allowFF); |
|
|
|
|
let extractor = await ext.fromUrl(url); |
|
|
|
|
extractor = require(`./extractors/${extractor}`); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log(`[extract] Starting "${url}"`, opt) |
|
|
|
|
if (opt.ignoreCache) opt.ignoreCache = parseBool(opt.ignoreCache); |
|
|
|
|
if (opt.allowCache) opt.allowCache = parseBool(opt.allowCache); |
|
|
|
|
if (opt.ignoreFF) opt.ignoreFF = parseBool(opt.ignoreFF); |
|
|
|
|
if (opt.allowFF) opt.allowFF = parseBool(opt.allowFF); |
|
|
|
|
|
|
|
|
|
if (config.db.active == true) { |
|
|
|
|
await waitUntilDbConnected(); |
|
|
|
|
if (opt.ignoreCache !== true) { |
|
|
|
|
if (config.debug == true) console.log("[db] Checking DB for desination..."); |
|
|
|
|
let f = await this.db.get(url); |
|
|
|
|
if (f !== null) { |
|
|
|
|
if (config.debug == true) console.log("[db] Found, sending solution..."); |
|
|
|
|
f.fromCache = true; |
|
|
|
|
f.fromFastforward = false; |
|
|
|
|
return f; |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log("[db] Not found, continuing..."); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
if (config.debug == true) console.log(`[extract] Starting "${url}"`, opt) |
|
|
|
|
|
|
|
|
|
if (config.fastforward == true && opt.ignoreFF !== true) { |
|
|
|
|
let r = await this.fastforward.get(url); |
|
|
|
|
if (r !== null) { |
|
|
|
|
f = { |
|
|
|
|
dateSolved: "unknown", |
|
|
|
|
originalUrl: url, |
|
|
|
|
destination: r, |
|
|
|
|
fromCache: false, |
|
|
|
|
fromFastforward: true, |
|
|
|
|
isURL: true |
|
|
|
|
}; |
|
|
|
|
if (config.db.active == true) { |
|
|
|
|
await waitUntilDbConnected(); |
|
|
|
|
if (opt.ignoreCache !== true) { |
|
|
|
|
if (config.debug == true) console.log("[db] Checking DB for desination..."); |
|
|
|
|
let f = await dbGet(url); |
|
|
|
|
if (f !== null) { |
|
|
|
|
if (config.debug == true) console.log("[db] Found, sending solution..."); |
|
|
|
|
f.fromCache = true; |
|
|
|
|
f.fromFastforward = false; |
|
|
|
|
return f; |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log("[db] Not found, continuing..."); |
|
|
|
|
} |
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
f = await extractor.get(this.fixSubdomain(url), opt); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log(`[extract] Finished "${url}", ${JSON.stringify(opt)} [Solution: ${(JSON.stringify(f) || f)}]`); |
|
|
|
|
|
|
|
|
|
if (typeof f == "string" || typeof f == "object" && !f.destinations) { |
|
|
|
|
if (!this.isUrl((f.destination || f)) || (f.destination || f) == url) { |
|
|
|
|
console.log((f.destination || f)); |
|
|
|
|
if (typeof (f.destination || f) == "string" && !extractor.hostnames.includes("linkvertise.com")) { |
|
|
|
|
if (config.debug == true) console.log("[extract] URL was invalid.", (f.destination||f), url); |
|
|
|
|
throw "Invalid URL from backend."; |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log("[extract] URL was invalid, but it was ignored because it was a Linkvertise link.", url); |
|
|
|
|
} |
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log("[extract] Got one link, proceeding..."); |
|
|
|
|
|
|
|
|
|
let iu = this.isUrl((f.destination || f)); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
let d = { |
|
|
|
|
destination: (f.destination || f), |
|
|
|
|
if (config.fastforward == true && opt.ignoreFF !== true) { |
|
|
|
|
let r = await ffGet(url); |
|
|
|
|
if (r !== null) { |
|
|
|
|
f = { |
|
|
|
|
dateSolved: "unknown", |
|
|
|
|
originalUrl: url, |
|
|
|
|
dateSolved: (new Date() * 1), |
|
|
|
|
isURL: iu |
|
|
|
|
destination: r, |
|
|
|
|
fromCache: false, |
|
|
|
|
fromFastforward: true, |
|
|
|
|
isURL: true |
|
|
|
|
}; |
|
|
|
|
return f; |
|
|
|
|
} |
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
f = await extractor.get(fixSubdomain(url), opt); |
|
|
|
|
if (config.debug == true) console.log(`[extract] Finished "${url}", ${JSON.stringify(opt)} [Solution: ${(JSON.stringify(f) || f)}]`); |
|
|
|
|
|
|
|
|
|
if (typeof f == "string" || typeof f == "object" && !f.destinations) { |
|
|
|
|
if (!isUrl((f.destination || f)) || (f.destination || f) == url) { |
|
|
|
|
if (typeof (f.destination || f) == "string" && !extractor.hostnames.includes("linkvertise.com")) { |
|
|
|
|
if (config.debug == true) console.log("[extract] URL was invalid.", (f.destination||f), url); |
|
|
|
|
throw "Invalid URL from backend."; |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log("[extract] URL was invalid, but it was ignored because it was a Linkvertise link.", url); |
|
|
|
|
} |
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (f.fastforward == true) { |
|
|
|
|
if (config.debug == true) console.log(`[extract] Detected FastForward response, correcting and sending...`); |
|
|
|
|
if (config.db.active == true) { |
|
|
|
|
if (opt.allowCache !== false) { |
|
|
|
|
if (config.debug == true) console.log("[db] Checking if data on this link already exists on the DB..."); |
|
|
|
|
let f = await links.findOne({"originalUrl": url}); |
|
|
|
|
if (f == null) f = await links.findOne({"original-url": url}); // older version compatibility
|
|
|
|
|
if (f == null) { |
|
|
|
|
if (config.debug == true) console.log(`[db] Nothing found. Adding FastForward solution to DB...`) |
|
|
|
|
await links.insertOne(d); |
|
|
|
|
if (config.debug == true) console.log(`[db] Added.`); |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log(`[db] Data already exists. Continuing correcting and sending...`); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
if (config.debug == true) console.log("[extract] Got one link, proceeding..."); |
|
|
|
|
|
|
|
|
|
d.dateSolved = "unknown"; |
|
|
|
|
d.fromCache = false; |
|
|
|
|
d.fromFastforward = true; |
|
|
|
|
return d; |
|
|
|
|
} |
|
|
|
|
let iu = isUrl((f.destination || f)); |
|
|
|
|
|
|
|
|
|
if (config.fastforward == true && opt?.allowFF !== false) { |
|
|
|
|
await this.fastforward.send(url, (f.destination || f)); |
|
|
|
|
} |
|
|
|
|
let d = { |
|
|
|
|
destination: (f.destination || f), |
|
|
|
|
originalUrl: url, |
|
|
|
|
dateSolved: (new Date() * 1), |
|
|
|
|
isURL: iu |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
if (f.fastforward == true) { |
|
|
|
|
if (config.debug == true) console.log(`[extract] Detected FastForward response, correcting and sending...`); |
|
|
|
|
if (config.db.active == true) { |
|
|
|
|
if (opt.allowCache !== false) { |
|
|
|
|
if (opt.ignoreCache == true) { |
|
|
|
|
if (config.debug == true) console.log(`[db] Replacing old version of "${url}" in DB.`) |
|
|
|
|
await links.findOneAndReplace({"originalUrl": url}, d); |
|
|
|
|
if (config.debug == true) console.log(`[db] Replaced.`) |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log(`[db] Adding to DB...`) |
|
|
|
|
if (config.debug == true) console.log("[db] Checking if data on this link already exists on the DB..."); |
|
|
|
|
let f = await links.findOne({"originalUrl": url}); |
|
|
|
|
if (f == null) f = await links.findOne({"original-url": url}); // older version compatibility
|
|
|
|
|
if (f == null) { |
|
|
|
|
if (config.debug == true) console.log(`[db] Nothing found. Adding FastForward solution to DB...`) |
|
|
|
|
await links.insertOne(d); |
|
|
|
|
if (config.debug == true) console.log(`[db] Added.`) |
|
|
|
|
if (config.debug == true) console.log(`[db] Added.`); |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log(`[db] Data already exists. Continuing correcting and sending...`); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
delete d._id; |
|
|
|
|
d.dateSolved = "unknown"; |
|
|
|
|
d.fromCache = false; |
|
|
|
|
d.fromFastforward = false; |
|
|
|
|
|
|
|
|
|
d.fromFastforward = true; |
|
|
|
|
return d; |
|
|
|
|
} else if (typeof f == "object" && f.destinations) { |
|
|
|
|
let d = { |
|
|
|
|
destinations: (f.destinations), |
|
|
|
|
originalUrl: url, |
|
|
|
|
dateSolved: (new Date() * 1) |
|
|
|
|
}; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
for (let a in d.destinations) { |
|
|
|
|
if (!this.isUrl(d.destinations[a])) delete d.destinations[a]; // removes non-urls
|
|
|
|
|
if (config.fastforward == true && opt?.allowFF !== false) { |
|
|
|
|
await ffSend(url, (f.destination || f)); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (config.db.active == true) { |
|
|
|
|
if (opt.allowCache !== false) { |
|
|
|
|
if (opt.ignoreCache == true) { |
|
|
|
|
if (config.debug == true) console.log(`[db] Replacing old version of "${url}" in DB.`) |
|
|
|
|
await links.findOneAndReplace({"originalUrl": url}, d); |
|
|
|
|
if (config.debug == true) console.log(`[db] Replaced.`) |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log(`[db] Adding to DB...`) |
|
|
|
|
await links.insertOne(d); |
|
|
|
|
if (config.debug == true) console.log(`[db] Added.`) |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
d.destinations = [...new Set(d.destinations)]; // removes duplicates
|
|
|
|
|
delete d._id; |
|
|
|
|
d.fromCache = false; |
|
|
|
|
d.fromFastforward = false; |
|
|
|
|
|
|
|
|
|
if (f.destinations.length == 0) { |
|
|
|
|
if (config.debug == true) console.log("[lib] Destination array size is 0, throwing error..."); |
|
|
|
|
throw "Invalid response from backend."; |
|
|
|
|
}
|
|
|
|
|
return d; |
|
|
|
|
} else if (typeof f == "object" && f.destinations) { |
|
|
|
|
let d = { |
|
|
|
|
destinations: (f.destinations), |
|
|
|
|
originalUrl: url, |
|
|
|
|
dateSolved: (new Date() * 1) |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
if (config.db.active == true) { |
|
|
|
|
if (opt.allowCache !== false) { |
|
|
|
|
d.destinations = JSON.stringify(d.destinations); |
|
|
|
|
if (opt.ignoreCache == true) { |
|
|
|
|
if (config.debug == true) console.log(`[db] Replacing old version of "${url}" in DB.`) |
|
|
|
|
await links.findOneAndReplace({"originalUrl": url}, d); |
|
|
|
|
if (config.debug == true) console.log(`[db] Replaced.`) |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log(`[db] Adding to DB...`) |
|
|
|
|
await links.insertOne(d); |
|
|
|
|
if (config.debug == true) console.log(`[db] Added.`) |
|
|
|
|
} |
|
|
|
|
d.destinations = JSON.parse(d.destinations); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
for (let a in d.destinations) { |
|
|
|
|
if (!isUrl(d.destinations[a])) delete d.destinations[a]; // removes non-urls
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
delete d._id; |
|
|
|
|
d.fromCache = false; |
|
|
|
|
d.fromFastforward = false; |
|
|
|
|
d.destinations = [...new Set(d.destinations)]; // removes duplicates
|
|
|
|
|
|
|
|
|
|
return d; |
|
|
|
|
} else { |
|
|
|
|
if (f.destinations.length == 0) { |
|
|
|
|
if (config.debug == true) console.log("[lib] Destination array size is 0, throwing error..."); |
|
|
|
|
throw "Invalid response from backend."; |
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (config.db.active == true) { |
|
|
|
|
if (opt.allowCache !== false) { |
|
|
|
|
d.destinations = JSON.stringify(d.destinations); |
|
|
|
|
if (opt.ignoreCache == true) { |
|
|
|
|
if (config.debug == true) console.log(`[db] Replacing old version of "${url}" in DB.`) |
|
|
|
|
await links.findOneAndReplace({"originalUrl": url}, d); |
|
|
|
|
if (config.debug == true) console.log(`[db] Replaced.`) |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log(`[db] Adding to DB...`) |
|
|
|
|
await links.insertOne(d); |
|
|
|
|
if (config.debug == true) console.log(`[db] Added.`) |
|
|
|
|
} |
|
|
|
|
d.destinations = JSON.parse(d.destinations); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} catch(err) { |
|
|
|
|
throw err; |
|
|
|
|
} |
|
|
|
|
},
|
|
|
|
|
solve: async function(sitekey, type, opt) { |
|
|
|
|
// opt value must be like:
|
|
|
|
|
// {
|
|
|
|
|
// "referer": "https://google.com"
|
|
|
|
|
// }
|
|
|
|
|
if (config.captcha.active == false) return null; |
|
|
|
|
if (config.captcha.service == "2captcha") { |
|
|
|
|
const tc = new two.Solver(config["captcha"]["key"]); |
|
|
|
|
let ref = (opt?.referer || "unknown location"); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log(`[captcha] Requesting CAPTCHA solve for a ${type} @ "${ref}"...`); |
|
|
|
|
let a; |
|
|
|
|
|
|
|
|
|
switch(type) { |
|
|
|
|
case "recaptcha": |
|
|
|
|
a = (await tc.recaptcha(sitekey, ref)).data; |
|
|
|
|
if (config.debug == true) console.log(`[captcha] Solved ${type} for "${ref}".`); |
|
|
|
|
return a;
|
|
|
|
|
case "hcaptcha": |
|
|
|
|
a = (await tc.hcaptcha(sitekey, ref)).data; |
|
|
|
|
if (config.debug == true) console.log(`[captcha] Solved ${type} for "${ref}".`); |
|
|
|
|
return a; |
|
|
|
|
case "image":
|
|
|
|
|
a = (await(tc.imageCaptcha(sitekey))).data; |
|
|
|
|
if (config.debug == true) console.log("[captcha] Solved image captcha.", a); |
|
|
|
|
return a; |
|
|
|
|
default: |
|
|
|
|
console.log(`[captcha] Invalid parameters were given to CAPTCHA solver.`) |
|
|
|
|
throw "Parameters for CAPTCHA solver are incorrect."; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
delete d._id; |
|
|
|
|
d.fromCache = false; |
|
|
|
|
d.fromFastforward = false; |
|
|
|
|
|
|
|
|
|
return d; |
|
|
|
|
} else { |
|
|
|
|
throw "Invalid response from backend."; |
|
|
|
|
} |
|
|
|
|
}, |
|
|
|
|
solveThroughPage: async function(p) { |
|
|
|
|
try { |
|
|
|
|
let type = await p.evaluate(function() { |
|
|
|
|
if (document.querySelector("iframe[title='recaptcha challenge expires in two minutes']") || document.querySelector(".g-recaptcha")) return "recaptcha"; |
|
|
|
|
else if (document.querySelector(".h-captcha")) return "hcaptcha"; |
|
|
|
|
else return null; |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
if (type == null) throw "Could not find CAPTCHA type."; |
|
|
|
|
if (config.debug == true) console.log("[captcha] Got CAPTCHA type:", type); |
|
|
|
|
|
|
|
|
|
let sk = await p.evaluate(function() { |
|
|
|
|
return ( |
|
|
|
|
document.querySelector("iframe[title='recaptcha challenge expires in two minutes']")?.src.split("k=")[1].split("&")[0] || |
|
|
|
|
document.querySelector(".g-recaptcha")?.getAttribute("data-sitekey") || |
|
|
|
|
document.querySelector(".h-captcha")?.getAttribute("data-sitekey") |
|
|
|
|
); |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log("[captcha] Got sitekey:", sk); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log("[captcha] Retrieved. Solving CAPTCHA..."); |
|
|
|
|
let c = await this.solve(sk, type, {referer: (await p.url())}); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log("[captcha] Solved CAPTCHA. Enterring solution and submitting form..."); |
|
|
|
|
await p.evaluate(`document.querySelector("[name='g-recaptcha-response']").value = "${c}";`); |
|
|
|
|
if (type == "hcaptcha") await p.evaluate(`document.querySelector("[name='h-captcha-response']").value = "${c}";`); |
|
|
|
|
} catch(err) { |
|
|
|
|
throw err; |
|
|
|
|
} catch(err) { |
|
|
|
|
throw err; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
function isUrl(url) { |
|
|
|
|
try { |
|
|
|
|
let u = new URL(url); |
|
|
|
|
if (u?.protocol !== null) return true; |
|
|
|
|
else return false; |
|
|
|
|
} catch(err) { |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
async function cfEmail(data) { |
|
|
|
|
if (!data.includes("/cdn-cgi/l/email-protection#")) data = `/cdn-cgi/l/email-protection#${data}`; |
|
|
|
|
let a = data; |
|
|
|
|
let s = data.indexOf(`/cdn-cgi/l/email-protection`); |
|
|
|
|
let m = data.length; |
|
|
|
|
|
|
|
|
|
if (a && s > -1 && m > 28) { |
|
|
|
|
var j = 28 + s; |
|
|
|
|
s = ''; |
|
|
|
|
if (j < m) { |
|
|
|
|
r = '0x' + a.substr(j, 2) | 0; |
|
|
|
|
for (j += 2; j < m && a.charAt(j) != 'X'; j += 2) s += '%' + ('0' + ('0x' + a.substr(j, 2) ^ r).toString(16)).slice(-2); |
|
|
|
|
j++; |
|
|
|
|
s = decodeURIComponent(s) + a.substr(j, m - j); |
|
|
|
|
} |
|
|
|
|
}, |
|
|
|
|
cookieString: function(co) { |
|
|
|
|
let s = ``; |
|
|
|
|
for (let c in co) { |
|
|
|
|
if (co[c].value == "deleted") continue; |
|
|
|
|
s = `${s} ${co[c].name}=${encodeURIComponent(co[c].value)};`; |
|
|
|
|
return s; |
|
|
|
|
} else { |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
async function cacheCount() { |
|
|
|
|
if (config.db?.active == false) return 0; |
|
|
|
|
if (links == undefined) await waitUntilDbConnected(); |
|
|
|
|
return (((await (await links.find({})).toArray()).length) || 0); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
async function dbGet(url) { |
|
|
|
|
if (links == undefined) await waitUntilDbConnected(); |
|
|
|
|
let f = await links.findOne({"originalUrl": url}); |
|
|
|
|
if (f !== null) { |
|
|
|
|
if (f.destinations) f.destinations = JSON.parse(f.destinations); |
|
|
|
|
if (f.destination && f.isURL == undefined || f.isURL == null) { |
|
|
|
|
f.isURL = require("./lib").isUrl(f.destination); |
|
|
|
|
} |
|
|
|
|
s = s.substring(0, s.length - 1); |
|
|
|
|
return s.substring(1); |
|
|
|
|
}, |
|
|
|
|
fastforward: { |
|
|
|
|
get: async function(url, igcb) { |
|
|
|
|
try { |
|
|
|
|
if ((isCrowdBypass(new URL(url).hostname) || igcb == true)) { |
|
|
|
|
let b = `domain=${new URL(url).hostname}&path=${new URL(url).pathname.substring(1)}${new URL(url).search}`; |
|
|
|
|
if (config.debug == true) console.log(`[fastforward] Made body content: `, b); |
|
|
|
|
if (config.debug == true) console.log("[fastforward] Checking FastForward crowd bypass..."); |
|
|
|
|
let d = await got({ |
|
|
|
|
method: "POST", |
|
|
|
|
url: "https://crowd.fastforward.team/crowd/query_v1", |
|
|
|
|
body: b, |
|
|
|
|
timeout: (5 * 1000), |
|
|
|
|
throwHttpErrors: false, // Prevent status errors
|
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log("[fastforward] Recieved response", d.status, d.data); |
|
|
|
|
|
|
|
|
|
if (d.status == 204) return null; |
|
|
|
|
else if (d.status == 200) return d.data; |
|
|
|
|
else { |
|
|
|
|
console.log(`[silent error] FastForward error:\nInvalid response code: ${d.status}\n${d.data}`); |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log("[fastforward] Tried to get FastForward URL, not acceptable URL."); |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
} catch(err) { |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
}, |
|
|
|
|
send: async function(url, dest, igcb) { |
|
|
|
|
try { |
|
|
|
|
if ((isCrowdBypass(new URL(url).hostname) || igcb == true)) { |
|
|
|
|
let b = `domain=${new URL(url).hostname}&path=${new URL(url).pathname.substring(1)}${new URL(url).search}&target=${encodeURIComponent(dest)}`; |
|
|
|
|
if (config.debug == true) console.log(`[fastforward] Made body content: `, b); |
|
|
|
|
if (config.debug == true) console.log("[fastforward] Sending to FastForward crowd bypass..."); |
|
|
|
|
|
|
|
|
|
let d = await got({ |
|
|
|
|
method: "POST", |
|
|
|
|
url: "https://crowd.fastforward.team/crowd/contribute_v1", |
|
|
|
|
timeout: (7 * 1000), |
|
|
|
|
body: b, |
|
|
|
|
throwHttpErrors: false // Prevent status errors
|
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log("[fastforward] Recieved response", d.status, d.data); |
|
|
|
|
|
|
|
|
|
if (d.status == 201) return true; |
|
|
|
|
else { |
|
|
|
|
console.log(`[silent error] FastForward error:\nInvalid response code: ${d.status}\n${d.data}`); |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
return f; |
|
|
|
|
} else { |
|
|
|
|
f = await links.findOne({"original-url": url}); // older version compatibility
|
|
|
|
|
if (f !== null) { |
|
|
|
|
delete f._id; |
|
|
|
|
if (f["date-solved"]) { |
|
|
|
|
if (config.debug == true) console.log("[db] Updating old solution format from DB..."); |
|
|
|
|
f = { |
|
|
|
|
dateSolved: f["date-solved"], |
|
|
|
|
originalUrl: f["original-url"], |
|
|
|
|
destination: f["destination"], |
|
|
|
|
isURL: require("./lib").isUrl(f["destination"]) |
|
|
|
|
} |
|
|
|
|
} catch(err) { |
|
|
|
|
return null; |
|
|
|
|
delete f._id; |
|
|
|
|
await links.findOneAndReplace({"original-url": url}, f); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
}, |
|
|
|
|
byteCount: function(string) {return encodeURI(string).split(/%..|./).length - 1;}, |
|
|
|
|
config: config, |
|
|
|
|
isUrl: function(url) { |
|
|
|
|
try { |
|
|
|
|
let u = new URL(url); |
|
|
|
|
if (u?.protocol !== null) return true; |
|
|
|
|
else return false; |
|
|
|
|
} catch(err) { |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
}, |
|
|
|
|
db: { |
|
|
|
|
get: async function(url) { |
|
|
|
|
if (links == undefined) await waitUntilDbConnected(); |
|
|
|
|
let f = await links.findOne({"originalUrl": url}); |
|
|
|
|
if (config.debug == true) console.log("[db] Sending DB response..."); |
|
|
|
|
delete f._id; |
|
|
|
|
return f;
|
|
|
|
|
} else if (f == null) { |
|
|
|
|
f = await links.findOne({"url": url}); // older version compatibility
|
|
|
|
|
if (f !== null) { |
|
|
|
|
if (f.destinations) f.destinations = JSON.parse(f.destinations); |
|
|
|
|
if (f.destination && f.isURL == undefined || f.isURL == null) { |
|
|
|
|
f.isURL = require("./lib").isUrl(f.destination); |
|
|
|
|
if (config.debug == true) console.log("[db] Updating old solution format from DB..."); |
|
|
|
|
f = { |
|
|
|
|
originalUrl: f["url"], |
|
|
|
|
destination: f["response"], |
|
|
|
|
dateSolved: (new Date(f.date) * 1) |
|
|
|
|
} |
|
|
|
|
delete f._id; |
|
|
|
|
await links.findOneAndReplace({"url": url}, f); |
|
|
|
|
delete f._id; |
|
|
|
|
return f; |
|
|
|
|
} else { |
|
|
|
|
f = await links.findOne({"original-url": url}); // older version compatibility
|
|
|
|
|
if (f !== null) { |
|
|
|
|
delete f._id; |
|
|
|
|
if (f["date-solved"]) { |
|
|
|
|
if (config.debug == true) console.log("[db] Updating old solution format from DB..."); |
|
|
|
|
f = { |
|
|
|
|
dateSolved: f["date-solved"], |
|
|
|
|
originalUrl: f["original-url"], |
|
|
|
|
destination: f["destination"], |
|
|
|
|
isURL: require("./lib").isUrl(f["destination"]) |
|
|
|
|
} |
|
|
|
|
delete f._id; |
|
|
|
|
await links.findOneAndReplace({"original-url": url}, f); |
|
|
|
|
} |
|
|
|
|
if (config.debug == true) console.log("[db] Sending DB response..."); |
|
|
|
|
delete f._id; |
|
|
|
|
return f;
|
|
|
|
|
} else if (f == null) { |
|
|
|
|
f = await links.findOne({"url": url}); // older version compatibility
|
|
|
|
|
if (f !== null) { |
|
|
|
|
if (config.debug == true) console.log("[db] Updating old solution format from DB..."); |
|
|
|
|
f = { |
|
|
|
|
originalUrl: f["url"], |
|
|
|
|
destination: f["response"], |
|
|
|
|
dateSolved: (new Date(f.date) * 1) |
|
|
|
|
} |
|
|
|
|
delete f._id; |
|
|
|
|
await links.findOneAndReplace({"url": url}, f); |
|
|
|
|
delete f._id; |
|
|
|
|
return f; |
|
|
|
|
} else { |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
}, |
|
|
|
|
removeTor: function(args) { |
|
|
|
|
for (let b in args?.args) { |
|
|
|
|
if (args?.args?.includes("proxy-server")) { |
|
|
|
|
let c = new URL(a[b].split("=")[1]); |
|
|
|
|
if ((c.hostname == "localhost" || c.hostname == "127.0.0.1") && c.port == "9050") { |
|
|
|
|
delete a[b]; |
|
|
|
|
if (config.debug == true) console.log("[lib] Removed Tor from Puppeteer arguments, due to compatibility issues."); |
|
|
|
|
} |
|
|
|
|
else continue;
|
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function fixSubdomain(url) { |
|
|
|
|
// function meant to be for sites that have you visit multiple different domains
|
|
|
|
|
|
|
|
|
|
let h = new URL(url).hostname; |
|
|
|
|
switch(h) { |
|
|
|
|
case "go.birdurls.com": |
|
|
|
|
case "birdurls.com": |
|
|
|
|
return `https://birdurls.com/${url.split("/").slice(3).join("/")}`; |
|
|
|
|
|
|
|
|
|
case "owllink.net": |
|
|
|
|
case "go.owllink.net": |
|
|
|
|
return `https://owllink.net/${url.split("/").slice(3).join("/")}`; |
|
|
|
|
|
|
|
|
|
case "crazyblog.in": |
|
|
|
|
case "open.crazyblog.in": |
|
|
|
|
case "redd.crazyblog.in": |
|
|
|
|
url = url.replace("open.crazyblog.in", "redd.crazyblog.in"); |
|
|
|
|
url = url.replace("//crazyblog.in", "//redd.crazyblog.in"); |
|
|
|
|
return url; |
|
|
|
|
|
|
|
|
|
case "medipost.org": |
|
|
|
|
case "links.medipost.org": |
|
|
|
|
case "usalink.io": |
|
|
|
|
return `https://links.medipost.org/${url.split("/").slice(3).join("/")}` |
|
|
|
|
|
|
|
|
|
case "pdiskshortener.in": |
|
|
|
|
return `https://1.htlinks.in/${url.split("/").slice(3).join("/")}`; |
|
|
|
|
|
|
|
|
|
default: return url; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
async function ffGet(url, igcb) { |
|
|
|
|
try { |
|
|
|
|
if ((isCrowdBypass(new URL(url).hostname) || igcb == true)) { |
|
|
|
|
let b = `domain=${new URL(url).hostname}&path=${new URL(url).pathname.substring(1)}${new URL(url).search}`; |
|
|
|
|
if (config.debug == true) console.log(`[fastforward] Made body content: `, b); |
|
|
|
|
if (config.debug == true) console.log("[fastforward] Checking FastForward crowd bypass..."); |
|
|
|
|
let d = await got({ |
|
|
|
|
method: "POST", |
|
|
|
|
url: "https://crowd.fastforward.team/crowd/query_v1", |
|
|
|
|
body: b, |
|
|
|
|
timeout: (5 * 1000), |
|
|
|
|
throwHttpErrors: false, // Prevent status errors
|
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log("[fastforward] Recieved response", d.status, d.data); |
|
|
|
|
|
|
|
|
|
if (d.status == 204) return null; |
|
|
|
|
else if (d.status == 200) return d.data; |
|
|
|
|
else { |
|
|
|
|
console.log(`[silent error] FastForward error:\nInvalid response code: ${d.status}\n${d.data}`); |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
} else { |
|
|
|
|
if (config.debug == true) console.log("[fastforward] Tried to get FastForward URL, not acceptable URL."); |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
return args; |
|
|
|
|
}, |
|
|
|
|
cloudflare: { |
|
|
|
|
email: function(data) { |
|
|
|
|
if (!data.includes("/cdn-cgi/l/email-protection#")) data = `/cdn-cgi/l/email-protection#${data}`; |
|
|
|
|
let a = data; |
|
|
|
|
let s = data.indexOf(`/cdn-cgi/l/email-protection`); |
|
|
|
|
let m = data.length; |
|
|
|
|
|
|
|
|
|
if (a && s > -1 && m > 28) { |
|
|
|
|
var j = 28 + s; |
|
|
|
|
s = ''; |
|
|
|
|
if (j < m) { |
|
|
|
|
r = '0x' + a.substr(j, 2) | 0; |
|
|
|
|
for (j += 2; j < m && a.charAt(j) != 'X'; j += 2) s += '%' + ('0' + ('0x' + a.substr(j, 2) ^ r).toString(16)).slice(-2); |
|
|
|
|
j++; |
|
|
|
|
s = decodeURIComponent(s) + a.substr(j, m - j); |
|
|
|
|
} |
|
|
|
|
return s; |
|
|
|
|
} else { |
|
|
|
|
} catch(err) { |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
async function ffSend(url, dest, igcb) { |
|
|
|
|
try { |
|
|
|
|
if ((isCrowdBypass(new URL(url).hostname) || igcb == true)) { |
|
|
|
|
let b = `domain=${new URL(url).hostname}&path=${new URL(url).pathname.substring(1)}${new URL(url).search}&target=${encodeURIComponent(dest)}`; |
|
|
|
|
if (config.debug == true) console.log(`[fastforward] Made body content: `, b); |
|
|
|
|
if (config.debug == true) console.log("[fastforward] Sending to FastForward crowd bypass..."); |
|
|
|
|
|
|
|
|
|
let d = await got({ |
|
|
|
|
method: "POST", |
|
|
|
|
url: "https://crowd.fastforward.team/crowd/contribute_v1", |
|
|
|
|
timeout: (7 * 1000), |
|
|
|
|
body: b, |
|
|
|
|
throwHttpErrors: false // Prevent status errors
|
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log("[fastforward] Recieved response", d.status, d.data); |
|
|
|
|
|
|
|
|
|
if (d.status == 201) return true; |
|
|
|
|
else { |
|
|
|
|
console.log(`[silent error] FastForward error:\nInvalid response code: ${d.status}\n${d.data}`); |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
}, |
|
|
|
|
cacheCount: async function() { |
|
|
|
|
if (config.db?.active == false) return 0; |
|
|
|
|
if (links == undefined) await waitUntilDbConnected(); |
|
|
|
|
return (((await (await links.find({})).toArray()).length) || 0); |
|
|
|
|
}, |
|
|
|
|
fixSubdomain: function(url) { |
|
|
|
|
// function meant to be for sites that have you visit multiple different domains
|
|
|
|
|
|
|
|
|
|
let h = new URL(url).hostname; |
|
|
|
|
switch(h) { |
|
|
|
|
case "go.birdurls.com": |
|
|
|
|
case "birdurls.com": |
|
|
|
|
return `https://birdurls.com/${url.split("/").slice(3).join("/")}`; |
|
|
|
|
|
|
|
|
|
case "owllink.net": |
|
|
|
|
case "go.owllink.net": |
|
|
|
|
return `https://owllink.net/${url.split("/").slice(3).join("/")}`; |
|
|
|
|
|
|
|
|
|
case "crazyblog.in": |
|
|
|
|
case "open.crazyblog.in": |
|
|
|
|
case "redd.crazyblog.in": |
|
|
|
|
url = url.replace("open.crazyblog.in", "redd.crazyblog.in"); |
|
|
|
|
url = url.replace("//crazyblog.in", "//redd.crazyblog.in"); |
|
|
|
|
return url; |
|
|
|
|
|
|
|
|
|
case "medipost.org": |
|
|
|
|
case "links.medipost.org": |
|
|
|
|
case "usalink.io": |
|
|
|
|
return `https://links.medipost.org/${url.split("/").slice(3).join("/")}` |
|
|
|
|
|
|
|
|
|
case "pdiskshortener.in": |
|
|
|
|
return `https://1.htlinks.in/${url.split("/").slice(3).join("/")}`; |
|
|
|
|
|
|
|
|
|
default: return url; |
|
|
|
|
} catch(err) { |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
async function solveCaptcha(sitekey, type, opt) { |
|
|
|
|
// opt value must be like:
|
|
|
|
|
// {
|
|
|
|
|
// "referer": "https://google.com"
|
|
|
|
|
// }
|
|
|
|
|
if (config.captcha.active == false) return null; |
|
|
|
|
if (config.captcha.service == "2captcha") { |
|
|
|
|
const tc = new two.Solver(config["captcha"]["key"]); |
|
|
|
|
let ref = (opt?.referer || "unknown location"); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log(`[captcha] Requesting CAPTCHA solve for a ${type} @ "${ref}"...`); |
|
|
|
|
let a; |
|
|
|
|
|
|
|
|
|
switch(type) { |
|
|
|
|
case "recaptcha": |
|
|
|
|
a = (await tc.recaptcha(sitekey, ref)).data; |
|
|
|
|
if (config.debug == true) console.log(`[captcha] Solved ${type} for "${ref}".`); |
|
|
|
|
return a;
|
|
|
|
|
case "hcaptcha": |
|
|
|
|
a = (await tc.hcaptcha(sitekey, ref)).data; |
|
|
|
|
if (config.debug == true) console.log(`[captcha] Solved ${type} for "${ref}".`); |
|
|
|
|
return a; |
|
|
|
|
case "image":
|
|
|
|
|
a = (await(tc.imageCaptcha(sitekey))).data; |
|
|
|
|
if (config.debug == true) console.log("[captcha] Solved image captcha.", a); |
|
|
|
|
return a; |
|
|
|
|
default: |
|
|
|
|
console.log(`[captcha] Invalid parameters were given to CAPTCHA solver.`) |
|
|
|
|
throw "Parameters for CAPTCHA solver are incorrect."; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
async function solveThroughPage(p) { |
|
|
|
|
try { |
|
|
|
|
let type = await p.evaluate(function() { |
|
|
|
|
if (document.querySelector("iframe[title='recaptcha challenge expires in two minutes']") || document.querySelector(".g-recaptcha")) return "recaptcha"; |
|
|
|
|
else if (document.querySelector(".h-captcha")) return "hcaptcha"; |
|
|
|
|
else return null; |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
if (type == null) throw "Could not find CAPTCHA type."; |
|
|
|
|
if (config.debug == true) console.log("[captcha] Got CAPTCHA type:", type); |
|
|
|
|
|
|
|
|
|
let sk = await p.evaluate(function() { |
|
|
|
|
return ( |
|
|
|
|
document.querySelector("iframe[title='recaptcha challenge expires in two minutes']")?.src.split("k=")[1].split("&")[0] || |
|
|
|
|
document.querySelector(".g-recaptcha")?.getAttribute("data-sitekey") || |
|
|
|
|
document.querySelector(".h-captcha")?.getAttribute("data-sitekey") |
|
|
|
|
); |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log("[captcha] Got sitekey:", sk); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log("[captcha] Retrieved. Solving CAPTCHA..."); |
|
|
|
|
let c = await solveCaptcha(sk, type, {referer: (await p.url())}); |
|
|
|
|
|
|
|
|
|
if (config.debug == true) console.log("[captcha] Solved CAPTCHA. Enterring solution and submitting form..."); |
|
|
|
|
await p.evaluate(`document.querySelector("[name='g-recaptcha-response']").value = "${c}";`); |
|
|
|
|
if (type == "hcaptcha") await p.evaluate(`document.querySelector("[name='h-captcha-response']").value = "${c}";`); |
|
|
|
|
} catch(err) { |
|
|
|
|
throw err; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
function cookieString(co) { |
|
|
|
|
let s = ``; |
|
|
|
|
for (let c in co) { |
|
|
|
|
if (co[c].value == "deleted") continue; |
|
|
|
|
s = `${s} ${co[c].name}=${encodeURIComponent(co[c].value)};`; |
|
|
|
|
} |
|
|
|
|
s = s.substring(0, s.length - 1); |
|
|
|
|
return s.substring(1); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
function byteCount(string) { |
|
|
|
|
return encodeURI(string).split(/%..|./).length - 1; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
function isCrowdBypass(host) { |
|
|
|
|
switch(host) { |
|
|
|
|
case /wadooo\.com|gotravelgo\.space|pantauterus\.me|liputannubi\.net/: |
|
|
|
|