const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
const fs = require('fs'); // Import the fs module to work with files

puppeteer.use(StealthPlugin());

(async () => {
    const browser = await puppeteer.launch({
            executablePath:'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
        headless: false,
        args: [
            "--no-sandbox",
            "--disable-setuid-sandbox",
            "--disable-blink-features=AutomationControlled"
        ]
    });

    const page = await browser.newPage();
    await page.setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36");

    let allData = [];

    for (let listPage = 1; listPage <= 5; listPage++) {
        console.log(`Fetching data from listpage=${listPage}...`);

        // Construct the URL with the changing listpage parameter
        const apiUrl = `https://easy.co.il/n/jsons/bizlist?version=2.3&c=268&client=web&listpage=${listPage}&lat=32.059925&lng=34.785126&rad=8905&mapid=0&viewport=desktop&lang=he`;

        // Navigate to the page
        await page.goto("https://easy.co.il/list/Hair-Design", { waitUntil: 'networkidle2' });

        // Wait for the page to load completely
        await new Promise(resolve => setTimeout(resolve, 3000));

        // Extract data from the API response
        const data = await page.evaluate((listPage) => {
            return fetch(`https://easy.co.il/n/jsons/bizlist?version=2.3&c=268&client=web&listpage=${listPage}&lat=32.059925&lng=34.785126&rad=8905&mapid=0&viewport=desktop&lang=he`)
                .then(response => response.json())
                .then(json => json.bizlist?.list || []) // Extract only the list
                .catch(err => console.log("Error fetching data:", err));
        });

        allData.push(...data); // Append the results to the array

        console.log(`✅ Retrieved ${data.length} businesses from page ${listPage}`);

        // Wait before making the next request to avoid being blocked
    }

    console.log(`🔹 Total businesses collected: ${allData.length}`);

    // Extract only required fields
    const formattedData = allData.map(biz => ({
        id: biz.id,
        city: biz.city,
        category: biz.category,
        bizname: biz.bizname,
        address: biz.address,
        phone: biz.phone
    }));

    // Write the formatted data to a file
    const filePath = 'formattedData.json';
    fs.writeFileSync(filePath, JSON.stringify(formattedData, null, 2), 'utf-8'); // Save as a JSON file
    console.log(`📌 Data saved to ${filePath}`);
    console.log(JSON.stringify(formattedData, null, 2));

    await browser.close();
})();