/* Fetches robots.txt and llms.txt for a ranked domain list, politely: two requests a site, one
   site per slot, an honest user agent. Appends one JSON line a site, so a rerun resumes.

     bun collect.ts <rank,domain csv> <out.jsonl> [concurrency] */

import { appendFileSync, existsSync, readFileSync } from 'node:fs'

const USER_AGENT = 'invisit-research/1.0 (+https://invisit.dev/bots; help@invisit.dev)'
const TIMEOUT_MS = 15_000
// RFC 9309 asks parsers to read at least 500 KiB.
const MAX_BYTES = 512 * 1024

const [listPath, outPath, slots = '30'] = process.argv.slice(2)
if (!listPath || !outPath) throw new Error('usage: collect.ts <list> <out> [concurrency]')

interface Fetched {
  readonly status: number
  readonly url: string
  readonly contentType: string
  readonly body: string
}

async function readCapped(response: Response): Promise<string> {
  const reader = response.body?.getReader()
  if (!reader) return ''
  const chunks: Uint8Array[] = []
  let total = 0
  while (total < MAX_BYTES) {
    const { done, value } = await reader.read()
    if (done) break
    chunks.push(value)
    total += value.byteLength
  }
  await reader.cancel().catch(() => undefined)
  return new TextDecoder().decode(Buffer.concat(chunks).subarray(0, MAX_BYTES))
}

async function get(url: string): Promise<Fetched | { readonly error: string }> {
  try {
    const response = await fetch(url, {
      headers: { 'User-Agent': USER_AGENT, Accept: 'text/plain, */*;q=0.5' },
      signal: AbortSignal.timeout(TIMEOUT_MS),
    })
    return {
      status: response.status,
      url: response.url,
      contentType: response.headers.get('content-type') ?? '',
      body: await readCapped(response),
    }
  } catch (error) {
    return { error: error instanceof Error ? `${error.name}: ${error.message}`.slice(0, 100) : 'unknown' }
  }
}

const isHtml = (page: Fetched) =>
  /html/i.test(page.contentType) || /^\s*<(!doctype|html|\?xml|head|body)/i.test(page.body)
const looksLikeRobots = (page: Fetched) =>
  page.status === 200 &&
  !isHtml(page) &&
  (page.body.trim() === '' ||
    /^\s*(user-agent|disallow|allow|sitemap|crawl-delay|host|content-signal)\s*:/im.test(page.body))

const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))

async function visit(rank: number, domain: string) {
  const attempts = [`https://${domain}/robots.txt`, `https://www.${domain}/robots.txt`]
  let robots: Fetched | undefined
  let lastStatus: number | null = null
  let lastError: string | null = null
  for (const url of attempts) {
    const page = await get(url)
    if ('error' in page) {
      lastError = page.error
      continue
    }
    lastStatus = page.status
    lastError = null
    if (looksLikeRobots(page)) {
      robots = page
      break
    }
    // A definite answer from the apex: the www host would say the same, spare it the request.
    if (page.status >= 400 && page.status < 500) break
  }

  let llms: Fetched | { readonly error: string } | null = null
  const origin = new URL(robots?.url ?? attempts[0]!).origin
  if (lastError === null) {
    await sleep(1_000)
    llms = await get(`${origin}/llms.txt`)
  }
  const llmsPage = llms && !('error' in llms) ? llms : null
  const hasLlms =
    llmsPage !== null &&
    llmsPage.status === 200 &&
    !isHtml(llmsPage) &&
    llmsPage.body.trim().length > 0

  return {
    rank,
    domain,
    fetched_at: new Date().toISOString(),
    status: lastStatus,
    error: lastError,
    origin,
    robots: robots?.body ?? null,
    llms_status: llmsPage?.status ?? null,
    llms_present: hasLlms,
    llms_spec_h1: hasLlms ? /^\s*#\s+\S/.test(llmsPage.body) : false,
    llms_bytes: hasLlms ? llmsPage.body.length : 0,
  }
}

const done = new Set<number>()
if (existsSync(outPath)) {
  for (const line of readFileSync(outPath, 'utf8').split('\n')) {
    if (line) done.add((JSON.parse(line) as { rank: number }).rank)
  }
}

const queue = readFileSync(listPath, 'utf8')
  .split('\n')
  .filter(Boolean)
  .map((line) => {
    const [rank, domain] = line.trim().split(',')
    return { rank: Number(rank), domain: domain! }
  })
  .filter((item) => !done.has(item.rank))

let index = 0
let finished = 0
const startedAt = Date.now()

await Promise.all(
  Array.from({ length: Number(slots) }, async () => {
    for (;;) {
      const item = queue[index++]
      if (!item) return
      appendFileSync(outPath, `${JSON.stringify(await visit(item.rank, item.domain))}\n`)
      finished += 1
      if (finished % 500 === 0)
        console.log(`${finished}/${queue.length} in ${Math.round((Date.now() - startedAt) / 1000)}s`)
    }
  }),
)
console.log(`done: ${finished}`)
process.exit(0)