gdd/lib.js

const axios = require("axios");
const cheerio = require("cheerio");
const two = require("2captcha");
const scp = require("set-cookie-parser");
const fs = require("fs");

let defaultHeaders = {
  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
  "Accept-Language": "en-US,en",
  "Connection": "keep-alive",
  "Sec-Fetch-Dest": "document",
  "Sec-Fetch-Mode": "navigate",
  "Sec-Fetch-Site": "cross-site",
  "TE": "trailers",
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:100.0) Gecko/20100101 Firefox/101.0"
}

module.exports = {
  defaultHeaders: defaultHeaders,
  isDriveLink: isDriveLink,
  getTypeLink: getTypeLink,
  fetchMetadata: fetchMetadata,
  fetchDownloadLink: fetchDownloadLink,
  returnWhenDone: returnWhenDone,
  toBool: toBool,
  isDebug: isDebug,
  flattenFolder: flattenFolder,
  getConfig: getConfig,
  setConfig: setConfig
}

async function isDriveLink(url) {
  try {
    url = new URL(url);
    let h = url.hostname;
    let p = url.pathname;
    if (h == "drive.google.com") {
      if (p.startsWith("/drive/folders/") || p.startsWith("/file/d/") || p == "/uc") {
        return true;
      } else {
        return false;
      }
    } else {
      return false;
    }
  } catch(e) {
    return false;
  }
}

function getTypeLink(url) {
  if (url.includes("folder")) return "folder";
  else return "file";
}

async function fetchMetadata(url, pathOffset, args, is_retry) {
  try {
    let headers = defaultHeaders;
    let type = getTypeLink(url);
  
    if (isDebug(args) == true) console.log(`[DEBUG] Got type`, type); 
  
    if (type == "file" && url.includes("uc?id")) url = `https://drive.google.com/file/d/${url.split("?id=")[1].split("&")[0].split("/")[0]}`;
  
    if (isDebug(args) == true) console.log(`[DEBUG] Sending request to`, url); 
    
    let string = ``;
    if (args["cookies"]) {
      let cookie = await getCookies(args["cookies"], "drive.google.com");
      
      for (let a in cookie) {
        string = `${string} ${cookie[a].name}=${cookie[a].value};`;
      }
      headers.Cookie = string.substring(1, (string.length - 1));
    }
  
    let resp = await axios({
      method: "GET",
      url: url,
      validateStatus: function() {return true},
      beforeRedirect: async function (options, {headers}) {
        if (headers.location) {
          let url = new URL(headers.location)
          if (url.hostname == "accounts.google.com") {
            throw new Error("Cannot access file/folder due to invalid permissions.");
          } else if (url.hostname == "www.google.com" && url.pathname == "/sorry/index") {
            if (isDebug(args) == true) console.log(`[DEBUG] Found sorry page, resolving...`);
          } else {
            if (isDebug(args) == true) console.log(`[DEBUG] Caught unknown redirect`, url.href);
          }
        }
      },
      headers: headers
    });
  
    if (isDebug(args) == true) console.log(`[DEBUG] Request sent, parsing...`); 
    let $ = cheerio.load(resp.data);

    if ($("#recaptcha").length > 0) {
      // sorry page handling

      if (!getConfig()?.captcha) {
        throw new Error("Got CAPTCHA, could not solve due to lack of credentials.");
      }

      let cookies = scp.parse(resp.headers["set-cookie"]);
      let sitekey = $("#recaptcha").attr("data-sitekey");
      let s = $("#recaptcha").attr("data-s");
      let q = $("input[name='q']").val();
      let con = $("input[name='continue']").val();
      let sorry = await solveSorry({
        ref: `https://www.google.com/sorry/index`,
        cookies: cookies,
        sitekey: sitekey,
        s: s,
        q: q,
        cont: con
      }, 0, args);

      $ = sorry.cheerio;
    }
  
    if (type == "folder") {
      if (isDebug(args) == true) console.log(`[DEBUG] Parsing as folder...`); 
      
      let data = {
        files: [],
        id: url.split("/folders/")[1].split("/")[0].split("?")[0],
        type: "folder"
      };
  
      $("#drive_main_page > div div[role='main'] > div > c-wiz > div[data-enable-upload-to-view] > c-wiz > div > c-wiz > div > c-wiz[data-bucketnames] > div > c-wiz > c-wiz > div > c-wiz > div").each(function(e) {
        let ele = $("#drive_main_page > div div[role='main'] > div > c-wiz > div[data-enable-upload-to-view] > c-wiz > div > c-wiz > div > c-wiz[data-bucketnames] > div > c-wiz > c-wiz > div > c-wiz > div")[e];
        if (ele?.attribs?.["data-id"]) {
          if (isDebug(args) == true) console.log(`[DEBUG] Got file id`, ele.attribs["data-id"]); 
          let name = $("#drive_main_page > div div[role='main'] > div > c-wiz > div[data-enable-upload-to-view] > c-wiz > div > c-wiz > div > c-wiz[data-bucketnames] > div > c-wiz > c-wiz > div > c-wiz > div [data-tooltip]")[e]?.attribs?.["data-tooltip"];
          ele.attribs
          data.files.push({
            id: ele?.attribs?.["data-id"],
            name: name,
            type: getTypeofFile("#drive_main_page > div div[role='main'] > div > c-wiz > div[data-enable-upload-to-view] > c-wiz > div > c-wiz > div > c-wiz[data-bucketnames] > div > c-wiz > c-wiz > div > c-wiz > div", e, $),
            path: `${pathOffset}`
          });
        }
      });
      
      data.name = $("title").text().split(" - Google Drive")
      data.name = data.name.slice(0, data.name.length - 1).join(" - Google Drive");
      if (isDebug(args) == true) console.log(`[DEBUG] Got folder name`, data.name);
      return data;
    } else {
      if ($("[itemprop='url']")) { 
        let data = {
          id: $("[itemprop='url']").attr("content").split("/d/")[1].split("/")[0],
          name: $("[itemprop='name']").attr("content"),
          type: "file",
        };
        if (isDebug(args) == true) console.log(`[DEBUG] Got file metadata`, data);
        return data;
      } else {
        return null;
      }
    }
  } catch(err) {
    throw err;
  }
}

async function fetchDownloadLink(id, args) {
  let headers = defaultHeaders;

  if (isDebug(args) == true) console.log(`[DEBUG] ID:`, id);
  if (isDebug(args) == true) console.log(`[DEBUG] Sending download request...`);
  try {
    let string = ``;
    if (args["cookies"]) {
      let cookie = await getCookies(args["cookies"], "drive.google.com");
      
      for (let a in cookie) {
        string = `${string} ${cookie[a].name}=${cookie[a].value};`;
      }
      headers.Cookie = string.substring(1, (string.length - 1));
    }

    let resp = await axios({
      method: "GET",
      beforeRedirect: function(options, {headers}) {
        if (headers.location.includes("googleusercontent")) {
          throw {code: "LT100", url: headers.location}
        }
      },
      url: `https://drive.google.com/u/0/uc?id=${id}&export=download`,
      headers: headers,
      maxContentSize: 2000
    });
    if (isDebug(args) == true) console.log(`[DEBUG] Parsing download request...`);
    let $ = cheerio.load(resp.data);

    if ($(".uc-error-subcaption")[0]) {
      let err = $(".uc-error-subcaption").text();
      throw new Error(err);
    }
  
    let url = $("form#downloadForm").attr("action");
    if (isDebug(args) == true) console.log(`[DEBUG] Got download URL:`, url);
    headers["Sec-Fetch-Dest"] = "document";
    headers["Sec-Fetch-Mode"] = "navigate";
    headers["Sec-Fetch-Site"] = "same-origin";
    headers["Sec-Fetch-User"] = "?1";
  
    try {
      if (isDebug(args) == true) console.log(`[DEBUG] Sending redirect request...`);
      
      let string = ``;
      if (args["cookies"]) {
        let cookie = await getCookies(args["cookies"], "drive.google.com");
        
        for (let a in cookie) {
          string = `${string} ${cookie[a].name}=${cookie[a].value};`;
        }
        headers.Cookie = string.substring(1, (string.length - 1));
      }
      
      resp = await axios({
        method: "POST",
        url: url,
        headers: headers,
        data: "",
        maxRedirects: 0
      });
  
      if (isDebug(args) == true) console.log(`[DEBUG] Got unexpected non-redirect response, throwing error.`);
      if (resp.headers) throw new Error("Download did not redirect properly.");
    } catch(err) {
      if (err.response?.headers?.location !== undefined) {
        if (isDebug(args) == true) console.log(`[DEBUG] Got expected redirect, URL:`, url);
        return err.response.headers.location;
      } else throw err;
    }
  } catch(err) {
    throw err;
  }
}

async function returnWhenDone(dl, pb) {
  return new Promise(function(resolve, reject) {
    try {
      dl.on("end", function() {
        pb.stop();
        resolve(true);
      });
    } catch(err) {
      reject(err);
    }
  });
}

function toBool(c) {
  if (c == undefined) return c;
  if (typeof c == "string") c = c.toLowerCase();
  switch(c) {
    case "y":
    case "ye":
    case "yes":
    case "t":
    case "tr":
    case "tru":
    case "true":
    case "1":
      return true;
    case "n":
    case "no":
    case "f":
    case "fa":
    case "fal":
    case "fals":
    case "false":
    case "0":
      return false;
    default:
      return c;
  }
}

function isDebug(args) {
  if (toBool(args.d) == true || toBool(args.debug) == true) {
    return true;
  } else {
    return false;
  }
}

function getTypeofFile(selector, index, $) {
  let label = $(`${selector} > div > [aria-label]`)[index]?.attribs?.["aria-label"]?.toLowerCase();
  if (label.includes("folder")) return "folder";
  else if (
    label == "google docs" || 
    label == "google slides" || 
    label == "google forms" ||
    label == "google sheets" || 
    label == "google drawings"
  ) return "document";
  else return "file";
}

async function flattenFolder(meta, folderOffset, args) {
  if (toBool(args["flatten"]) == false) {
    if (isDebug(args) == true) console.log(`[DEBUG] Got option to not flatten folder, removing folders and documents...`);
    for (let a in meta.files) {
      if (meta.files[a].type !== "file") delete meta.files[a];
    }
    meta.files = meta.files.filter(obj => typeof obj == "object")
    return meta;
  }

  console.log("- Flattening folder", meta.id);
  if (meta.files) {
    for (let a in meta.files) {
      if (meta.files[a].type == "folder") {
        let id = meta.files[a].id;

        if (folderOffset == "") folderOffset = `./${meta.files[a].name}`;
        else folderOffset = `${folderOffset}/${meta.files[a].name}`;
        
        let data = await fetchMetadata(`https://drive.google.com/drive/folders/${id}`, folderOffset, args);
        
        for (let b in data.files) {
          meta.files.push(data.files[b]);
        }

        delete meta.files[a];
      } else if (meta.files[a].type == "document") {
        if (isDebug(args) == true) console.log(`[DEBUG] Removed ${meta.files[a]} because of the unsupported file type "document".`);
        delete meta.files[a];
      }
    }
    
    let f = meta.files.filter(e => e.type == "folder");
    meta.files = meta.files.filter(obj => typeof obj == "object")
    if (f.length !== 0) return (await flattenFolder(meta, folderOffset, args));
    else return meta;
  }
}

async function getCookies(path, domain) {
  if (!fs.existsSync(path)) throw new Error("Cookies file does not exist");
  else {
    let file = fs.readFileSync(path).toString();

    file = file.split(`\n`);

    for (let a in file) {
      file[a] = file[a].split(`\t`);
    }

    if (file[0][0].startsWith("#")) file = file.slice(4)

    for (let a in file) {
      if (file[a][0] !== domain) delete file[a];
      else {
        file[a] = {
          domain: file[a][0],
          httpOnly: toBool(file[a][1]),
          path: file[a][2],
          secure: toBool(file[a][3]),
          expires: new Date(file[a][4] * 1000),
          name: file[a][5],
          value: file[a][6]
        }
      }
    }
  
    file = file.filter(element => {
      if (Object.keys(element).length !== 0) {
        return true;
      }
    
      return false;
    });
    return file;
  }
}

function getConfig() {
  let config_location = `${__dirname}/config.json`;

  if (fs.existsSync(config_location)) {
    return JSON.parse(fs.readFileSync(config_location).toString());
  } else {
    return {};
  }
}

function setConfig(name, value) {
  let config_location = `${__dirname}/config.json`;
  let config = getConfig();

  config[name] = value;

  fs.writeFileSync(config_location, JSON.stringify(config, null, 2));
}

async function solveSorry(obj, attempt, args) {
  if (attempt == 0) console.log("Solving rate-limit page, give us a moment...");
  else {
    switch (attempt) {
      case 1:
        console.log(`or two...`);
        break;
      case 2:
        console.log(`or three...`);
        break;
      case 3: 
        console.log(`or four...`);
        break;
      case 4:
        console.log(`or five...`);
        break;
      default:
        throw new Error("Could not solve sorry CAPTCHA challenge.");
    }
    
  }
  attempt = (attempt + 1);
  if (isDebug(args) == true) console.log(`[DEBUG] CAPTCHA solve attempt ${attempt}`);

  /* 
    objects *should* look like:
    {
      ref: "url to page",
      sitekey: "page sitekey",
      s: "data-s from sorry page, if any",
      cookies: [object of cookies],
      q: "value of q in form",
      cont: "value of continue in form"
    }
  */

  let config = getConfig();
  let solver = config.captcha?.solver;
  let key = config.captcha?.key;

  if (solver && key) {
    let res;
    switch(solver) {
      case "2captcha":
        if (isDebug(args) == true) console.log(`[DEBUG] Sending 2captcha request to solve the CAPTCHA...`);
        let tc = new two.Solver(key);
        res = await tc.recaptcha(obj.sitekey, obj.ref, {
          "data-s": obj.s
        });
        res = res.data;
      break;

      default:
        throw new Error("Solver is not available.");
    }

    if (typeof res !== "string") throw new Error("Solver did not return a proper response.");
    // res *must* be a string
        if (isDebug(args) == true) console.log(`[DEBUG] Got response:`, res);

    let req_data = `g-recaptcha-response=${res}&q=${obj.q}&continue=${obj.cont}`

    let headers = defaultHeaders;
    headers.Referer = obj.ref;
    headers["Sec-Fetch-Dest"] = "document";
    headers["Sec-Fetch-Mode"] = "navigate";
    headers["Sec-Fetch-Site"] = "same-origin";
    headers["Sec-Fetch-User"] = "?1";
    headers["Content-Type"] = "application/x-www-form-urlencoded";


    if (isDebug(args) == true) console.log(`[DEBUG] Sending /sorry solve request...`);
    let resp = await axios({
      method: "POST",
      data: req_data,
      headers: headers,
      validateStatus: function() {return true;},
      url: `https://www.google.com/sorry/index`
    });

    let $ = cheerio.load(resp.data);
    if (isDebug(args) == true) console.log(`[DEBUG] Parsing response...`);

    if ($("#recaptcha").length > 0) return (await solveSorry(obj, attempt, args));
    else return {cheerio: $, cookies: scp.parse(resp.headers["set-cookie"])};
  } else {
    throw new Error("Unable to solve, invalid credentials");
  }
}