import { chromium } from "playwright"; import { readFileSync, writeFileSync, existsSync } from "fs"; const chunkId = process.argv[2]; if (!chunkId && chunkId !== "0") { console.error("Usage: node scrape-payee-chunk.mjs "); process.exit(1); } const PO_LIST = `output/not-found-2026-chunk${chunkId}.json`; const OUTPUT_FILE = `output/payee-2026-chunk${chunkId}-results.json`; const PROGRESS_FILE = `output/payee-2026-chunk${chunkId}-progress.json`; const PROFILE_DIR = `chrome-profile-${chunkId}`; const DELAY_MS = 800; const allPOs = JSON.parse(readFileSync(PO_LIST, "utf-8")); console.log(`[Chunk ${chunkId}] Loaded ${allPOs.length} POs`); let completed = {}; if (existsSync(PROGRESS_FILE)) { completed = JSON.parse(readFileSync(PROGRESS_FILE, "utf-8")); console.log(`[Chunk ${chunkId}] Resuming: ${Object.keys(completed).length} already done`); } const remaining = allPOs.filter((po) => !completed[po]); console.log(`[Chunk ${chunkId}] Remaining: ${remaining.length} POs\n`); if (remaining.length === 0) { console.log("All POs already scraped!"); process.exit(0); } function poUrl(po) { return `https://payeecentral.amazon.com/PurchaseOrder/Details?poNumber=${po}`; } // Copy chrome-profile if this chunk's profile doesn't exist import { execSync } from "child_process"; const profilePath = new URL(`./${PROFILE_DIR}`, import.meta.url).pathname; const mainProfile = new URL("./chrome-profile", import.meta.url).pathname; if (!existsSync(profilePath)) { console.log(`Copying chrome profile to ${PROFILE_DIR}...`); execSync(`cp -R "${mainProfile}" "${profilePath}"`); } const context = await chromium.launchPersistentContext(profilePath, { headless: false, channel: "chrome", args: ["--disable-blink-features=AutomationControlled"], ignoreDefaultArgs: ["--enable-automation"], }); const page = context.pages()[0] || (await context.newPage()); console.log(`[Chunk ${chunkId}] Opening Payee Central...`); await page.goto(poUrl(remaining[0]), { waitUntil: "networkidle", timeout: 30000 }); if (page.url().includes("signin") || page.url().includes("/ap/")) { console.log(`\n[Chunk ${chunkId}] 🔐 Login required. Please sign in...`); await page.waitForURL( (u) => { const s = u.toString(); return s.includes("payeecentral.amazon.com") && !s.includes("signin") && !s.includes("/ap/"); }, { timeout: 180000 } ); console.log(`[Chunk ${chunkId}] ✅ Logged in!\n`); } let results = []; if (existsSync(OUTPUT_FILE)) { results = JSON.parse(readFileSync(OUTPUT_FILE, "utf-8")); } let scraped = 0; let errors = 0; let consecutiveErrors = 0; const startTime = Date.now(); for (const po of remaining) { try { await page.goto(poUrl(po), { waitUntil: "networkidle", timeout: 20000 }); if (page.url().includes("signin") || page.url().includes("/ap/")) { console.log(`\n[Chunk ${chunkId}] 🔐 Session expired. Please log in again...`); await page.waitForURL( (u) => !u.toString().includes("signin") && !u.toString().includes("/ap/"), { timeout: 180000 } ); await page.goto(poUrl(po), { waitUntil: "networkidle", timeout: 20000 }); } await page.waitForTimeout(500); const bodyText = await page.innerText("body"); const shipToMatch = bodyText.match(/Ship To\s+([\s\S]*?)(?=Payee Central Account)/); const shipTo = shipToMatch ? shipToMatch[1].trim() : null; let siteCode = null; if (shipTo) { for (const re of [ /\(([A-Z0-9]{3,5})\)/, /(?:LLC|Inc)\s*-\s*([A-Z0-9]{3,5})\b/, /LLC\s+([A-Z0-9]{3,5})\b/, /^([A-Z0-9]{3,5})\s*-\s*Amazon/m, /Amazon\s+Fresh\s+([A-Z0-9]{3,5})\b/, ]) { const m = shipTo.match(re); if (m) { siteCode = m[1]; break; } } } const lineItems = await page.$$eval( "table tbody tr", (rows) => rows.map((row) => { const cells = row.querySelectorAll("td"); if (cells.length < 7) return null; return { description: cells[1]?.textContent?.trim() || null, need_by: cells[2]?.textContent?.trim() || null, quantity: cells[3]?.textContent?.trim() || null, unit: cells[4]?.textContent?.trim() || null, price: cells[5]?.textContent?.trim() || null, amount: cells[6]?.textContent?.trim() || null, }; }).filter(Boolean) ).catch(() => []); results.push({ po_number: po, site_code: siteCode, ship_to_raw: shipTo, line_items: lineItems, }); completed[po] = true; scraped++; consecutiveErrors = 0; const elapsed = (Date.now() - startTime) / 1000; const rate = scraped / elapsed; const eta = Math.round((remaining.length - scraped) / rate); const etaMin = Math.floor(eta / 60); const etaSec = eta % 60; process.stdout.write( `\r[C${chunkId}] [${scraped}/${remaining.length}] ${po} | ${siteCode || "?"} | ETA: ${etaMin}m${etaSec}s ` ); if (scraped % 10 === 0) { writeFileSync(OUTPUT_FILE, JSON.stringify(results, null, 2)); writeFileSync(PROGRESS_FILE, JSON.stringify(completed)); } } catch (err) { console.error(`\n[C${chunkId}] [${po}] Error: ${err.message}`); errors++; consecutiveErrors++; if (consecutiveErrors > 10) { console.error("Too many consecutive errors, stopping."); break; } } await page.waitForTimeout(DELAY_MS); } writeFileSync(OUTPUT_FILE, JSON.stringify(results, null, 2)); writeFileSync(PROGRESS_FILE, JSON.stringify(completed)); console.log(`\n\n[Chunk ${chunkId}] Done! Scraped: ${scraped}, Errors: ${errors}`); await context.close();