crawler.js

const [_, __, prefix, token] = process.argv;

import fetch from "node-fetch"

import fs from "fs/promises"

async function save(entry) {
    console.log(entry.title)
    const response = await fetch(`${prefix}/read/${entry.value}`, {
        headers: { token }
    })

    const article = await response.json()

    const { subject, content, url } = article.data

    await fs.writeFile(`articles/${url}.md`, `#${subject}\n\n ${content}`)
}

async function crawler() {
    const response = await fetch(`${prefix}/menu`, {
        headers: {
            token
        }
    })

    const menu = await response.json()

    let entries = menu.data

    while(entries.length > 0) {
        let entry = entries.shift()
        if (entry.value) {
            await save(entry);
        }
        entries = entries.concat(entry.children)
    }

    console.log(JSON.stringify(menu))
}

crawler();