#!/usr/bin/env node
/**
 * Scroll the page down to trigger infinite scroll / lazy loading.
 *
 * Scrolls down 1 page at a time, up to INFINISCROLL_SCROLL_LIMIT times,
 * ensuring at least INFINISCROLL_MIN_HEIGHT (default 16,000px) is reached.
 * Stops early if no new content loads after a scroll.
 *
 * Optionally expands <details> elements and clicks "load more" buttons.
 *
 * Usage: on_Snapshot__45_infiniscroll.js --url=<url>
 * Output: JSONL with scroll stats (no files created)
 *
 * Environment variables:
 *     INFINISCROLL_ENABLED: Enable/disable (default: true)
 *     INFINISCROLL_TIMEOUT: Max timeout in seconds (default: 120)
 *     INFINISCROLL_SCROLL_DELAY: Delay between scrolls in ms (default: 2000)
 *     INFINISCROLL_SCROLL_DISTANCE: Pixels per scroll (default: 1600)
 *     INFINISCROLL_SCROLL_LIMIT: Max scroll iterations (default: 10)
 *     INFINISCROLL_MIN_HEIGHT: Min page height to reach in px (default: 16000)
 *     INFINISCROLL_EXPAND_DETAILS: Expand <details> and comments (default: true)
 */

const fs = require("fs");
const path = require("path");
const { loadConfig } = require("../base/utils.js");

const PLUGIN_DIR = path.basename(__dirname);
const hookConfig = loadConfig();
const SNAP_DIR = path.resolve((hookConfig.SNAP_DIR || ".").trim());
const OUTPUT_DIR = path.join(SNAP_DIR, PLUGIN_DIR);
if (!fs.existsSync(OUTPUT_DIR)) {
  fs.mkdirSync(OUTPUT_DIR, { recursive: true });
}
process.chdir(OUTPUT_DIR);

// Import generic helpers from base/utils.js
const {
  ensureNodeModuleResolution,
  getEnvBool,
  getEnvInt,
  parseArgs,
  emitArchiveResultRecord,
} = require("../base/utils.js");
ensureNodeModuleResolution(module);

// Import chrome-specific utilities from chrome_utils.js
const {
  connectToPage,
  resolvePuppeteerModule,
} = require("../chrome/chrome_utils.js");

// Check if infiniscroll is enabled BEFORE requiring puppeteer
if (!getEnvBool("INFINISCROLL_ENABLED", true)) {
  console.error("Skipping infiniscroll (INFINISCROLL_ENABLED=False)");
  emitArchiveResultRecord("skipped", "INFINISCROLL_ENABLED=False");
  process.exit(0);
}

const puppeteer = resolvePuppeteerModule();

const PLUGIN_NAME = "infiniscroll";
const CHROME_SESSION_DIR = "../chrome";

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

/**
 * Expand <details> elements and click "load more" buttons for comments.
 * Based on archivebox.ts expandComments function.
 */
async function expandDetails(page, options = {}) {
  const { timeout = 30000, limit = 500, delay = 500 } = options;

  const startTime = Date.now();

  // First, expand all <details> elements
  const detailsExpanded = await page.evaluate(() => {
    let count = 0;
    // Generic <details> elements
    document.querySelectorAll("details:not([open])").forEach((el) => {
      el.open = true;
      count++;
    });
    // Github README details sections
    document.querySelectorAll("article details:not([open])").forEach((el) => {
      el.open = true;
      count++;
    });
    // Github issue discussion hidden comments
    document
      .querySelectorAll(
        "div.js-discussion details:not(.details-overlay):not([open])"
      )
      .forEach((el) => {
        el.open = true;
        count++;
      });
    // HedgeDoc/Markdown details sections
    document
      .querySelectorAll(".markdown-body details:not([open])")
      .forEach((el) => {
        el.open = true;
        count++;
      });
    return count;
  });

  if (detailsExpanded > 0) {
    console.error(`Expanded ${detailsExpanded} <details> elements`);
  }

  // Then click "load more" buttons for comments
  const numExpanded = await page.evaluate(
    async ({ timeout, limit, delay }) => {
      // Helper to find elements by XPath
      function getElementsByXPath(xpath) {
        const results = [];
        const xpathResult = document.evaluate(
          xpath,
          document,
          null,
          XPathResult.ORDERED_NODE_ITERATOR_TYPE,
          null
        );
        let node;
        while ((node = xpathResult.iterateNext()) != null) {
          results.push(node);
        }
        return results;
      }

      const wait = (ms) => new Promise((res) => setTimeout(res, ms));

      // Find all "load more" type buttons/links
      const getLoadMoreLinks = () => [
        // Reddit (new)
        ...document.querySelectorAll("faceplate-partial[loading=action]"),
        // Reddit (old) - show more replies
        ...document.querySelectorAll('a[onclick^="return morechildren"]'),
        // Reddit (old) - show hidden replies
        ...document.querySelectorAll('a[onclick^="return togglecomment"]'),
        // Twitter/X - show more replies
        ...getElementsByXPath("//*[text()='Show more replies']"),
        ...getElementsByXPath("//*[text()='Show replies']"),
        // Generic "load more" / "show more" buttons
        ...getElementsByXPath("//*[contains(text(),'Load more')]"),
        ...getElementsByXPath("//*[contains(text(),'Show more')]"),
        // Hacker News
        ...document.querySelectorAll("a.morelink"),
      ];

      let expanded = 0;
      let loadMoreLinks = getLoadMoreLinks();
      const startTime = Date.now();

      while (loadMoreLinks.length > 0) {
        for (const link of loadMoreLinks) {
          // Skip certain elements
          if (link.slot === "children") continue;

          try {
            link.scrollIntoView({ behavior: "smooth" });
            link.click();
            expanded++;
            await wait(delay);
          } catch (e) {
            // Ignore click errors
          }

          // Check limits
          if (expanded >= limit) return expanded;
          if (Date.now() - startTime >= timeout) return expanded;
        }

        // Check for new load more links after clicking
        await wait(delay);
        loadMoreLinks = getLoadMoreLinks();
      }

      return expanded;
    },
    { timeout, limit, delay }
  );

  if (numExpanded > 0) {
    console.error(`Clicked ${numExpanded} "load more" buttons`);
  }

  return {
    detailsExpanded,
    commentsExpanded: numExpanded,
    total: detailsExpanded + numExpanded,
  };
}

async function scrollDown(page, options = {}) {
  const {
    timeout = 120000,
    scrollDelay = 2000,
    scrollDistance = 1600,
    scrollLimit = 10,
    minHeight = 16000,
    expandedTotal = 0,
  } = options;

  const startTime = Date.now();

  // Get page height using multiple methods (some pages use different scroll containers)
  const getPageHeight = () =>
    page.evaluate(() => {
      return Math.max(
        document.body.scrollHeight || 0,
        document.body.offsetHeight || 0,
        document.documentElement.scrollHeight || 0,
        document.documentElement.offsetHeight || 0
      );
    });

  const startingHeight = await getPageHeight();
  let lastHeight = startingHeight;
  let scrollCount = 0;
  let scrollPosition = 0;

  console.error(`Initial page height: ${startingHeight}px`);
  console.log(
    `Scrolled ${startingHeight.toLocaleString()}px | expanded ${expandedTotal.toLocaleString()} sections`
  );

  // Scroll to top first
  await page.evaluate(() => {
    window.scrollTo({ top: 0, left: 0, behavior: "smooth" });
  });
  await sleep(500);

  while (scrollCount < scrollLimit) {
    // Check timeout
    const elapsed = Date.now() - startTime;
    if (elapsed >= timeout) {
      console.error(`Timeout reached after ${scrollCount} scrolls`);
      break;
    }

    scrollPosition = (scrollCount + 1) * scrollDistance;
    console.error(
      `Scrolling down ${
        scrollCount + 1
      }x ${scrollDistance}px... (${scrollPosition}/${lastHeight})`
    );

    await page.evaluate((yOffset) => {
      window.scrollTo({ top: yOffset, left: 0, behavior: "smooth" });
    }, scrollPosition);

    scrollCount++;
    await sleep(scrollDelay);

    // Check if new content was added (infinite scroll detection)
    const newHeight = await getPageHeight();
    const addedPx = newHeight - lastHeight;

    if (addedPx > 0) {
      console.error(
        `Detected infini-scrolling: ${lastHeight}+${addedPx} => ${newHeight}`
      );
    } else if (scrollPosition >= newHeight + scrollDistance) {
      // Reached the bottom
      if (scrollCount > 2) {
        console.error(`Reached bottom of page at ${newHeight}px`);
        break;
      }
    }

    lastHeight = newHeight;
    console.log(
      `Scrolled ${lastHeight.toLocaleString()}px | expanded ${expandedTotal.toLocaleString()} sections`
    );

    // Check if we've reached minimum height and can stop
    if (lastHeight >= minHeight && scrollPosition >= lastHeight) {
      console.error(`Reached minimum height target (${minHeight}px)`);
      break;
    }
  }

  // Scroll to absolute bottom
  if (scrollPosition < lastHeight) {
    await page.evaluate(() => {
      window.scrollTo({
        top: document.documentElement.scrollHeight,
        left: 0,
        behavior: "smooth",
      });
    });
    await sleep(scrollDelay);
  }

  // Scroll back to top
  console.error(
    `Reached bottom of page at ${lastHeight}px, scrolling back to top...`
  );
  await page.evaluate(() => {
    window.scrollTo({ top: 0, left: 0, behavior: "smooth" });
  });
  await sleep(scrollDelay);

  const totalElapsed = Date.now() - startTime;

  return {
    scrollCount,
    finalHeight: lastHeight,
    startingHeight,
    elapsedMs: totalElapsed,
  };
}

async function main() {
  const args = parseArgs();
  const url = args.url;

  if (!url) {
    console.error("Usage: on_Snapshot__45_infiniscroll.js --url=<url>");
    process.exit(1);
  }

  const timeout = getEnvInt("INFINISCROLL_TIMEOUT", 120) * 1000;
  const scrollDelay = getEnvInt("INFINISCROLL_SCROLL_DELAY", 2000);
  const scrollDistance = getEnvInt("INFINISCROLL_SCROLL_DISTANCE", 1600);
  const scrollLimit = getEnvInt("INFINISCROLL_SCROLL_LIMIT", 10);
  const minHeight = getEnvInt("INFINISCROLL_MIN_HEIGHT", 16000);
  const expandDetailsEnabled = getEnvBool("INFINISCROLL_EXPAND_DETAILS", true);

  let browser = null;
  try {
    const connectTimeoutMs = Math.min(timeout, getEnvInt("TIMEOUT", 30) * 1000);
    const connection = await connectToPage({
      chromeSessionDir: CHROME_SESSION_DIR,
      timeoutMs: connectTimeoutMs,
      waitForNavigationComplete: true,
      postLoadDelayMs: 200,
      puppeteer,
    });
    browser = connection.browser;
    const page = connection.page;

    console.error(`Starting infinite scroll on ${url}`);
    console.log("Scrolled 0px | expanded 0 sections | scrolling...");

    // Expand <details> and comments before scrolling (if enabled)
    let expandResult = { total: 0, detailsExpanded: 0, commentsExpanded: 0 };
    if (expandDetailsEnabled) {
      console.error("Expanding <details> and comments...");
      expandResult = await expandDetails(page, {
        timeout: Math.min(timeout / 4, 30000),
        limit: 500,
        delay: scrollDelay / 4,
      });
    }

    const result = await scrollDown(page, {
      timeout,
      scrollDelay,
      scrollDistance,
      scrollLimit,
      minHeight,
      expandedTotal: expandResult.total,
    });

    // Expand again after scrolling (new content may have loaded)
    if (expandDetailsEnabled) {
      const expandResult2 = await expandDetails(page, {
        timeout: Math.min(timeout / 4, 30000),
        limit: 500,
        delay: scrollDelay / 4,
      });
      expandResult.total += expandResult2.total;
      expandResult.detailsExpanded += expandResult2.detailsExpanded;
      expandResult.commentsExpanded += expandResult2.commentsExpanded;
    }

    browser.disconnect();

    const finalHeightStr = result.finalHeight.toLocaleString();
    const outputStr = `scrolled ${finalHeightStr}px`;

    console.error(`Success: ${outputStr}`);
    emitArchiveResultRecord("succeeded", outputStr);
    process.exit(0);
  } catch (e) {
    if (browser) browser.disconnect();
    console.error(`ERROR: ${e.name}: ${e.message}`);
    process.exit(1);
  }
}

main().catch((e) => {
  console.error(`Fatal error: ${e.message}`);
  process.exit(1);
});
