293 lines
8.8 KiB
JavaScript
293 lines
8.8 KiB
JavaScript
|
import { chromium } from "playwright";
|
||
|
|
import { readFileSync, writeFileSync, existsSync } from "fs";
|
||
|
|
|
||
|
|
const EMAIL = process.env.COUPA_EMAIL;
|
||
|
|
const PASSWORD = process.env.COUPA_PASSWORD;
|
||
|
|
|
||
|
|
if (!EMAIL || !PASSWORD) {
|
||
|
|
console.error("Set COUPA_EMAIL and COUPA_PASSWORD env vars");
|
||
|
|
process.exit(1);
|
||
|
|
}
|
||
|
|
|
||
|
|
const SUPPLIER_ID = "895025";
|
||
|
|
const INSTANCE_URL = "https%3A%2F%2Famazon.coupahost.com%2F";
|
||
|
|
const OUTPUT_FILE = "output/scraped-pos.json";
|
||
|
|
const PROGRESS_FILE = "output/progress.json";
|
||
|
|
const DELAY_MS = 1500;
|
||
|
|
const KEEPALIVE_EVERY = 75;
|
||
|
|
|
||
|
|
const poListFile = process.argv[2];
|
||
|
|
if (!poListFile) {
|
||
|
|
console.error("Usage: node scrape.mjs <po-list.json>");
|
||
|
|
process.exit(1);
|
||
|
|
}
|
||
|
|
|
||
|
|
const allPOs = JSON.parse(readFileSync(poListFile, "utf-8"));
|
||
|
|
console.log(`Loaded ${allPOs.length} POs to scrape`);
|
||
|
|
|
||
|
|
let completed = {};
|
||
|
|
if (existsSync(PROGRESS_FILE)) {
|
||
|
|
completed = JSON.parse(readFileSync(PROGRESS_FILE, "utf-8"));
|
||
|
|
console.log(`Resuming: ${Object.keys(completed).length} already scraped`);
|
||
|
|
}
|
||
|
|
|
||
|
|
const remaining = allPOs.filter((po) => !completed[po]);
|
||
|
|
console.log(`Remaining: ${remaining.length} POs\n`);
|
||
|
|
|
||
|
|
if (remaining.length === 0) {
|
||
|
|
console.log("All POs already scraped!");
|
||
|
|
process.exit(0);
|
||
|
|
}
|
||
|
|
|
||
|
|
function poUrl(po) {
|
||
|
|
const numeric = po.split("-").slice(1).join("-");
|
||
|
|
return `https://supplier.coupahost.com/orders/${numeric}?instance_url=${INSTANCE_URL}&supplier_id=${SUPPLIER_ID}&`;
|
||
|
|
}
|
||
|
|
|
||
|
|
function isLoginPage(url) {
|
||
|
|
return url.includes("sessions/new") || url.includes("login") || url.includes("sign_in");
|
||
|
|
}
|
||
|
|
|
||
|
|
async function autoLogin(page) {
|
||
|
|
console.log("\n🔐 Session expired — auto-filling login form...");
|
||
|
|
|
||
|
|
// Step 1: Fill email and click Continue
|
||
|
|
try {
|
||
|
|
await page.waitForSelector("input#email", { state: "visible", timeout: 5000 });
|
||
|
|
await page.fill("input#email", EMAIL);
|
||
|
|
await page.click("button.s-login").catch(() => {});
|
||
|
|
console.log(" Email filled, clicked Continue.");
|
||
|
|
} catch {
|
||
|
|
// Already past email step
|
||
|
|
}
|
||
|
|
|
||
|
|
// Step 2: CAPTCHA appears before password — wait for user to solve it
|
||
|
|
// After CAPTCHA, password field will appear
|
||
|
|
console.log(" ⏳ Solve the CAPTCHA — password will auto-fill after...");
|
||
|
|
try {
|
||
|
|
await page.waitForSelector("input#password", { state: "visible", timeout: 120000 });
|
||
|
|
await page.fill("input#password", PASSWORD);
|
||
|
|
console.log(" Password filled, clicking Login...");
|
||
|
|
page.click("button.s-login").catch(() => {});
|
||
|
|
} catch {
|
||
|
|
console.log(" Password field never appeared. Check the browser.");
|
||
|
|
}
|
||
|
|
|
||
|
|
// Wait for login to complete (may need another CAPTCHA or just redirects)
|
||
|
|
console.log(" ⏳ Waiting for login to complete...");
|
||
|
|
try {
|
||
|
|
await page.waitForURL((url) => !isLoginPage(url.toString()), { timeout: 120000 });
|
||
|
|
console.log(" ✅ Login successful!\n");
|
||
|
|
return true;
|
||
|
|
} catch {
|
||
|
|
console.error(" ❌ Login timed out after 120s. Exiting.");
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
async function keepAlive(page) {
|
||
|
|
try {
|
||
|
|
await page.goto("https://supplier.coupahost.com/home/", {
|
||
|
|
waitUntil: "domcontentloaded",
|
||
|
|
timeout: 15000,
|
||
|
|
});
|
||
|
|
if (isLoginPage(page.url())) return false;
|
||
|
|
return true;
|
||
|
|
} catch {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
async function scrapePO(page, po) {
|
||
|
|
await page.goto(poUrl(po), { waitUntil: "networkidle", timeout: 30000 });
|
||
|
|
|
||
|
|
if (isLoginPage(page.url())) {
|
||
|
|
const ok = await autoLogin(page);
|
||
|
|
if (!ok) return null;
|
||
|
|
// Navigate to PO after re-login
|
||
|
|
await page.goto(poUrl(po), { waitUntil: "networkidle", timeout: 30000 });
|
||
|
|
if (isLoginPage(page.url())) return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
await page.waitForSelector("iframe#enterprise_frame", { timeout: 15000 });
|
||
|
|
await page.waitForTimeout(1500);
|
||
|
|
|
||
|
|
const frame =
|
||
|
|
page.frame({ name: "enterprise_frame" }) ||
|
||
|
|
page.frames().find((f) => f.url().includes("supplier_order_headers"));
|
||
|
|
|
||
|
|
if (!frame) return "no_iframe";
|
||
|
|
|
||
|
|
await frame.waitForLoadState("networkidle");
|
||
|
|
|
||
|
|
const data = await frame.evaluate(() => {
|
||
|
|
const text = (sel) => {
|
||
|
|
const el = document.querySelector(sel);
|
||
|
|
return el ? el.textContent.trim() : null;
|
||
|
|
};
|
||
|
|
|
||
|
|
const lineItems = [];
|
||
|
|
const rows = document.querySelectorAll(".coupa_datatable_row");
|
||
|
|
for (const row of rows) {
|
||
|
|
const desc = row.querySelector(".s-description");
|
||
|
|
const price = row.querySelector(".s-price span[title]");
|
||
|
|
const total = row.querySelector(".s-total span[title]");
|
||
|
|
const needBy = row.querySelector("[id$='_need_by_date']");
|
||
|
|
lineItems.push({
|
||
|
|
description: desc ? desc.textContent.trim() : null,
|
||
|
|
price: price ? price.textContent.trim() : null,
|
||
|
|
total: total ? total.textContent.trim() : null,
|
||
|
|
need_by: needBy ? needBy.textContent.trim() : null,
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
const addrEl = document.querySelector(
|
||
|
|
"label[for='order_header_ship_to_address'] + span.address, span.address"
|
||
|
|
);
|
||
|
|
let shipTo = null;
|
||
|
|
if (addrEl) {
|
||
|
|
const clone = addrEl.cloneNode(true);
|
||
|
|
clone.querySelectorAll(".form_element").forEach((el) => el.remove());
|
||
|
|
shipTo = clone.innerHTML
|
||
|
|
.replace(/<br\s*\/?>/gi, "\n")
|
||
|
|
.replace(/<[^>]+>/g, "")
|
||
|
|
.trim();
|
||
|
|
}
|
||
|
|
|
||
|
|
return {
|
||
|
|
status: text(".s-readable_status"),
|
||
|
|
order_date: text(".s-local_header_created_at"),
|
||
|
|
requester: text(".s-requester"),
|
||
|
|
payment_term: text("#order_header_payment_term .data"),
|
||
|
|
ship_to_raw: shipTo,
|
||
|
|
line_items: lineItems,
|
||
|
|
};
|
||
|
|
});
|
||
|
|
|
||
|
|
let site_code = null;
|
||
|
|
if (data.ship_to_raw) {
|
||
|
|
const m = data.ship_to_raw.match(/\(([A-Z0-9]{3,5})\)/);
|
||
|
|
if (m) site_code = m[1];
|
||
|
|
if (!site_code) {
|
||
|
|
const m2 = data.ship_to_raw.match(/Attn:\s*([A-Z0-9]{3,5})\b/);
|
||
|
|
if (m2) site_code = m2[1];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return { po_number: po, site_code, ...data };
|
||
|
|
}
|
||
|
|
|
||
|
|
// --- Main ---
|
||
|
|
// Use a persistent Chrome profile so reCAPTCHA trusts the browser
|
||
|
|
const userDataDir = new URL("./chrome-profile", import.meta.url).pathname;
|
||
|
|
const context = await chromium.launchPersistentContext(userDataDir, {
|
||
|
|
headless: false,
|
||
|
|
channel: "chrome",
|
||
|
|
args: ["--disable-blink-features=AutomationControlled"],
|
||
|
|
ignoreDefaultArgs: ["--enable-automation"],
|
||
|
|
});
|
||
|
|
const page = context.pages()[0] || (await context.newPage());
|
||
|
|
|
||
|
|
// Initial login
|
||
|
|
console.log("Opening Coupa supplier portal...");
|
||
|
|
await page.goto(poUrl(remaining[0]), { waitUntil: "networkidle", timeout: 30000 });
|
||
|
|
|
||
|
|
if (isLoginPage(page.url())) {
|
||
|
|
const ok = await autoLogin(page);
|
||
|
|
if (!ok) {
|
||
|
|
await context.close();
|
||
|
|
process.exit(1);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
let results = [];
|
||
|
|
if (existsSync(OUTPUT_FILE)) {
|
||
|
|
results = JSON.parse(readFileSync(OUTPUT_FILE, "utf-8"));
|
||
|
|
}
|
||
|
|
|
||
|
|
let scraped = 0;
|
||
|
|
let errors = 0;
|
||
|
|
let consecutiveErrors = 0;
|
||
|
|
const startTime = Date.now();
|
||
|
|
|
||
|
|
for (const po of remaining) {
|
||
|
|
// Session keep-alive
|
||
|
|
if (scraped > 0 && scraped % KEEPALIVE_EVERY === 0) {
|
||
|
|
process.stdout.write("\n Refreshing session...");
|
||
|
|
const alive = await keepAlive(page);
|
||
|
|
if (!alive) {
|
||
|
|
console.log(" session expired, re-logging in...");
|
||
|
|
await page.goto("https://supplier.coupahost.com/sessions/new", {
|
||
|
|
waitUntil: "networkidle",
|
||
|
|
});
|
||
|
|
const ok = await autoLogin(page);
|
||
|
|
if (!ok) break;
|
||
|
|
} else {
|
||
|
|
process.stdout.write(" ok\n");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
try {
|
||
|
|
const result = await scrapePO(page, po);
|
||
|
|
|
||
|
|
if (result === null) {
|
||
|
|
console.error(`\nFailed to authenticate. Saving progress.`);
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (result === "no_iframe") {
|
||
|
|
console.error(`\n [${po}] No iframe found, skipping`);
|
||
|
|
errors++;
|
||
|
|
consecutiveErrors++;
|
||
|
|
if (consecutiveErrors > 10) {
|
||
|
|
console.error("Too many consecutive errors, stopping.");
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
results.push(result);
|
||
|
|
completed[po] = true;
|
||
|
|
scraped++;
|
||
|
|
consecutiveErrors = 0;
|
||
|
|
|
||
|
|
const elapsed = (Date.now() - startTime) / 1000;
|
||
|
|
const rate = scraped / elapsed;
|
||
|
|
const eta = Math.round((remaining.length - scraped) / rate);
|
||
|
|
const etaMin = Math.floor(eta / 60);
|
||
|
|
const etaSec = eta % 60;
|
||
|
|
const desc0 = result.line_items[0]?.description?.substring(0, 50) || "no lines";
|
||
|
|
|
||
|
|
process.stdout.write(
|
||
|
|
`\r[${scraped}/${remaining.length}] ${po} | ${result.site_code || "?"} | ${desc0}... | ETA: ${etaMin}m${etaSec}s `
|
||
|
|
);
|
||
|
|
|
||
|
|
// Save every 10 POs (more frequent for monitor)
|
||
|
|
if (scraped % 10 === 0) {
|
||
|
|
writeFileSync(OUTPUT_FILE, JSON.stringify(results, null, 2));
|
||
|
|
writeFileSync(PROGRESS_FILE, JSON.stringify(completed));
|
||
|
|
}
|
||
|
|
} catch (err) {
|
||
|
|
console.error(`\n [${po}] Error: ${err.message}`);
|
||
|
|
errors++;
|
||
|
|
consecutiveErrors++;
|
||
|
|
|
||
|
|
if (consecutiveErrors > 10) {
|
||
|
|
console.error("Too many consecutive errors, stopping.");
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
await page.waitForTimeout(DELAY_MS);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Final save
|
||
|
|
writeFileSync(OUTPUT_FILE, JSON.stringify(results, null, 2));
|
||
|
|
writeFileSync(PROGRESS_FILE, JSON.stringify(completed));
|
||
|
|
|
||
|
|
console.log(`\n\nDone! Scraped: ${scraped}, Errors: ${errors}`);
|
||
|
|
console.log(`Total in output: ${results.length} POs`);
|
||
|
|
console.log(`Results saved to ${OUTPUT_FILE}`);
|
||
|
|
|
||
|
|
await context.close();
|