Skip to content

Instantly share code, notes, and snippets.

@inca

inca/slider-captcha.ts

Created March 10, 2020 17:24
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save inca/62b6d2ccad4aca3e9e3d6704d71a9e39 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save inca/62b6d2ccad4aca3e9e3d6704d71a9e39 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. inca created this gist Mar 10, 2020.
    189 changes: 189 additions & 0 deletions slider-captcha.ts
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,189 @@
    import { Element, Ctx } from '@ubio/engine';

    export async function captchaSlider(el: Element, ctx: Ctx) {
    const page = el.page;

    // Obtain elements to interact with
    const sliderEl = (await el.queryOne('.yidun_jigsaw', false))!;
    const imageEl = (await el.queryOne('.yidun_bg-img', false))!;

    // Get image resources (base64) so that we can send them for offscreen canvas processing
    const sliderSrc: string = await sliderEl.evaluateJson(el => el.src);
    const imageSrc: string = await imageEl.evaluateJson(el => el.src);
    const sliderRes = await page.send('Page.getResourceContent', {
    frameId: page.target.targetId,
    url: sliderSrc
    });
    const imageRes = await page.send('Page.getResourceContent', {
    frameId: page.target.targetId,
    url: imageSrc
    });

    // Interesting part: send images back to page to process and evaluate the offset
    const answer = await page.evaluateJson(async (sliderDataB64: string, imageDataB64: string) => {
    // Note: slider is always png, image is always jpg (so this might not work on other websites)
    const sliderData = getImageData(await loadImageBase64(sliderDataB64, 'png'));
    const imageData = getImageData(await loadImageBase64(imageDataB64, 'jpg'));
    const { width, height } = imageData;
    const maxDistance = width - sliderData.width;

    const canvas = document.createElement('canvas');
    canvas.width = width;
    canvas.height = height;
    // Uncomment for debugging:
    // document.body.appendChild(canvas);
    const ctx = canvas.getContext('2d')!;
    ctx.fillStyle = 'rgb(255,0,0)';
    ctx.putImageData(imageData, 0, 0, 0, 0, width, height);

    const sobelData = calcSobel(imageData);
    const points = findFeaturePoints(sliderData);
    const best = findAnswer(sobelData, points, maxDistance);

    // ctx.putImageData(sliderData, best.pos, 0, 0, 0, width, height);
    // Uncomment to visualize feature points
    // for (const [x, y] of points) {
    // ctx.fillRect(best.pos + x, y, 1, 1,);
    // }

    return { width, height, pos: best.pos };

    function loadImageBase64(base64: string, ext: string): Promise<HTMLImageElement> {
    return new Promise((resolve, reject) => {
    const img = document.createElement('img');
    img.src = `data:image/${ext};base64,${base64}`;
    img.onload = () => {
    resolve(img);
    };
    img.onerror = err => reject(err);
    });
    }

    function getImageData(img: HTMLImageElement) {
    const canvas = document.createElement('canvas');
    const { width, height } = img;
    canvas.width = width;
    canvas.height = height;
    const ctx = canvas.getContext('2d')!;
    ctx.drawImage(img, 0, 0);
    return ctx.getImageData(0, 0, width, height);
    }

    function findFeaturePoints(sliderData: ImageData): Array<[number, number]> {
    const { data, width, height } = sliderData;
    const points: Array<[number, number]> = [];
    for (let y = 1; y < height - 1; y += 2) {
    for (let x = 1; x < width - 1; x += 2) {
    let val = 0;
    for (const ox of [-1, 0, 1]) {
    for (const oy of [-1, 0, 1]) {
    const i = getIndex(width, x + ox, y + oy);
    const a = data[i + 3];
    val += a >= 255 ? 1 : 0;
    }
    }
    if (val > 1 && val < 8) {
    points.push([x, y]);
    }
    }
    }
    return points;
    }

    function calcSobel(imageData: ImageData): ImageData {
    const Kx = [
    [-1, 0, +1],
    [-2, 0, +2],
    [-1, 0, +1]
    ];
    const Ky = [
    [-1, -2, -1],
    [0, 0, 0],
    [+1, +2, +1]
    ];
    const { width, height, data } = imageData;
    const output = ctx.createImageData(width, height);
    for (let y = 1; y < height - 1; y += 1) {
    for (let x = 1; x < width - 1; x += 1) {
    let vx = 0;
    let vy = 0;
    for (let oy of [-1, 0, 1]) {
    for (let ox of [-1, 0, 1]) {
    const i = getIndex(width, x + ox, y + oy);
    const l = lum(data[i], data[i + 1], data[i + 2]);
    const kx = Kx[oy + 1][ox + 1];
    const ky = Ky[oy + 1][ox + 1];
    vx += l * kx;
    vy += l * ky;
    }
    }
    const val = Math.hypot(vx, vy);
    const i = getIndex(width, x, y);
    output.data[i + 0] = val;
    output.data[i + 1] = val;
    output.data[i + 2] = val;
    output.data[i + 3] = 255;
    }
    }
    return output;
    }

    function findAnswer(sobelData: ImageData, points: Array<[number, number]>, maxDistance: number) {
    const { width, data } = sobelData;
    const best = {
    pos: 0,
    val: 0,
    };
    for (let pos = 0; pos < maxDistance; pos++) {
    let val = 0;
    for (const [x, y] of points) {
    const i = getIndex(width, pos + x, y);
    val += data[i];
    }
    if (val > best.val) {
    best.val = val;
    best.pos = pos;
    }
    }
    return best;
    }

    function getIndex(width: number, x: number, y: number) {
    return (y * width + x) * 4;
    }

    function lum(r: number, g: number, b: number) {
    return 0.2126 * r + 0.7152 * g + 0.0722 * b;
    }

    }, sliderRes.content, imageRes.content);

    // Ok, that was a mouthful!
    // We now have 'pos' which is the distance in pixels we need to move the slider,
    // but it's in "original image space", so we have to transform it into "element space".
    // Plus, there appears to be some easing when
    const { width, pos } = answer;
    const imageBox = await imageEl.remote.getBoxModel();
    const offset = Math.round(pos * imageBox.width / width);

    // Now let's carefully move the slider element with CDP events
    const { x, y } = await sliderEl.remote.getStablePoint();
    await page.inputManager.mouseMove(x, y);
    await page.inputManager.mouseDown(x, y);
    for (let dx = 0; dx < offset; dx += 5) {
    const dy = Math.random() * 20 - 10;
    await page.inputManager.mouseMove(x + dx, y + dy);
    await new Promise(r => setTimeout(r, 5));
    }
    // Ok so here's the deal: there appears to be some easing on dragging,
    // so if we drag the exact `offset` pixels to the right, it will still
    // stay not aligned.
    // To fix this we need to figure the correction value between offset
    // and current left style.
    const left = await sliderEl.evaluateJson(el => Math.round(Number(el.style.left.replace('px', ''))));
    const diff = offset - left;
    const dy = Math.random() * 20 - 10;
    await page.inputManager.mouseMove(x + offset + diff, y + dy);
    await page.inputManager.mouseUp(x + offset + diff, y + dy);

    }