如何在 Crawlee 中用 Node.js SDK 处理验证码

# One install for the crawler, the browser and the solver client.
npm install crawlee playwright capskip

# Crawlee drives a real browser, so fetch one.
npx playwright install chromium

// npm install crawlee playwright capskip
const { PlaywrightCrawler } = require('crawlee');
const { CapSkip } = require('capskip');

// Local mode. Point host at a server IP to share one solver.
const solver = new CapSkip({ host: '127.0.0.1', port: 8080 });

async function solveIfChallenged(page, url, log) {
  const widget = page.locator('[data-sitekey]').first();
  if ((await widget.count()) === 0) return false;

  const sitekey = await widget.getAttribute('data-sitekey');
  log.info(`Solving sitekey ${sitekey}`);
  const result = await solver.recaptcha(sitekey, url);
  return result.code;   // the token
}

// The widget writes into a hidden textarea. Do the same.
await page.evaluate((token) => {
  const field = document.getElementById('g-recaptcha-response');
  field.value = token;
}, token);

// Then submit exactly as the page would, and wait for the result.
await Promise.all([
  page.waitForNavigation(),
  page.click('button[type=submit]'),
]);

// npm install crawlee playwright capskip
const crawler = new PlaywrightCrawler({
  // 60 is the default and it is shorter than a v2 solve plus a submit.
  requestHandlerTimeoutSecs: 180,

  // Three tries per URL, which is Crawlee's default and the right one.
  maxRequestRetries: 3,

  async requestHandler({ page, request, log }) {
    // your handler
  },
});

// npm install crawlee playwright capskip
const { PlaywrightCrawler, Dataset } = require('crawlee');
const { CapSkip } = require('capskip');

const solver = new CapSkip({ host: '127.0.0.1', port: 8080 });

const crawler = new PlaywrightCrawler({
  requestHandlerTimeoutSecs: 180,
  maxRequestRetries: 3,

  async requestHandler({ page, request, log }) {
    const widget = page.locator('[data-sitekey]').first();

    if ((await widget.count()) > 0) {
      const sitekey = await widget.getAttribute('data-sitekey');
      const result = await solver.recaptcha(sitekey, request.loadedUrl);

      await page.evaluate((token) => {
        document.getElementById('g-recaptcha-response').value = token;
      }, result.code);

      await Promise.all([
        page.waitForNavigation(),
        page.click('button[type=submit]'),
      ]);
      log.info(`Cleared the challenge on ${request.loadedUrl}`);
    }

    await Dataset.pushData({ url: request.loadedUrl, title: await page.title() });
  },
});

await crawler.run(['https://example.com/page-with-recaptcha']);

// npm install capskip
const { ApiException, NetworkException, TimeoutException } = require('capskip');

const crawler = new PlaywrightCrawler({
  requestHandlerTimeoutSecs: 180,

  // Runs between retries, while attempts remain.
  errorHandler({ request, log }, error) {
    log.warning(`Retry ${request.retryCount} for ${request.url}: ${error.message}`);
  },

  // Runs once, after the last attempt fails.
  failedRequestHandler({ request, log }) {
    log.error(`Gave up on ${request.url}`);
  },
});

// npm install capskip
const { CapSkip } = require('capskip');

// Same client, different address. Nothing else in the code changes.
const solver = new CapSkip({
  host: process.env.CAPSKIP_HOST || '127.0.0.1',
  port: Number(process.env.CAPSKIP_PORT || 8080),
});