Skip to content

Instantly share code, notes, and snippets.

@JHWelch

JHWelch/BinaryTestDivide.php

Created June 27, 2023 14:02
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save JHWelch/a59e0f1515c4c92e465f1a6c4b5125ad to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save JHWelch/a59e0f1515c4c92e465f1a6c4b5125ad to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. JHWelch created this gist Jun 27, 2023.
    200 changes: 200 additions & 0 deletions BinaryTestDivide.php
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,200 @@
    <?php

    namespace App\Console\Commands;

    use Illuminate\Console\Command;
    use Illuminate\Support\Collection;
    use Illuminate\Support\Str;
    use ReflectionClass;
    use Symfony\Component\Process\Process;

    class BinaryTestDivide extends Command
    {
    /**
    * The name and signature of the console command.
    *
    * @var string
    */
    protected $signature = 'test:divide {seed}';

    /**
    * The console command description.
    *
    * @var string
    */
    protected $description = 'Run tests in two groups until the test causing failures is found.';

    protected string $seed;

    /**
    * Execute the console command.
    *
    * @return int
    */
    public function handle()
    {
    $tests = $this->failingTests();

    $failingTest = $tests->shift();

    return $this->runBisect($failingTest, $tests);
    }

    protected function runBisect(string $failingTest, Collection $tests): int
    {
    [$first, $second] = $tests->split(2);

    $firstResult = $this->runTests($failingTest, $first, 'First Half Tests');
    $secondResult = $this->runTests($failingTest, $second, 'Second Half Tests');

    $this->table(['First', 'Second'], [
    [$firstResult ? '✅' : '❌', $secondResult ? '✅' : '❌'],
    ]);

    if ($firstResult && $secondResult) {
    $this->error('Both halves passed, try again');

    return Command::FAILURE;
    } elseif (!$firstResult && !$secondResult) {
    $this->error('Both halves failed, try again');

    return Command::FAILURE;
    }

    if ($this->checkTests($firstResult, $first)
    || $this->checkTests($secondResult, $second)) {

    return Command::SUCCESS;
    }

    $this->info('Running new bisect');
    $this->table(['First', 'Second'], [
    [$first->count(), $second->count()],
    ]);

    if (!$firstResult) {
    return $this->runBisect($failingTest, $first);
    } else {
    return $this->runBisect($failingTest, $second);
    }
    }

    protected function runTests(string $failingTest, Collection $tests, string $title): bool
    {
    $toRun = (clone $tests)->push($failingTest);
    $this->table([$title], $toRun->map(fn ($test) => [$test])->toArray());
    $process = new Process([
    $this->phpUnitPath(),
    '--random-order-seed', $this->seed(),
    '--stop-on-failure',
    '--filter', $toRun->map(fn ($test) => addslashes($test))->implode('|'),
    ], timeout: 300);

    $process->run();

    foreach ($process as $type => $data) {
    echo $data;
    }

    $process->wait();

    return $process->getExitCode() === self::SUCCESS;

    }

    /**
    * @param Collection<string> $tests
    */
    protected function checkTests(bool $result, Collection $tests): bool
    {
    if ($result || $tests->count() !== 1) {
    return false;
    }

    $this->info('Found the Troublesome Test');
    $this->info($tests->first());
    if ($classPath = $this->classPath($tests->first())) {
    $this->info($classPath);
    }

    return true;
    }

    protected function classPath(string $test): ?string
    {
    if (!class_exists($test)) {
    return null;
    }

    return (new ReflectionClass($test))->getFileName();
    }

    protected function failingTests(): Collection
    {
    $process = new Process([
    $this->phpUnitPath(),
    '--random-order-seed', $this->seed(),
    '--stop-on-failure',
    '--debug',
    ], timeout: 300);

    $process->start();

    $lines = [];

    foreach ($process as $type => $data) {
    $lines[] = $data;
    echo $data;
    }

    $process->wait();

    if ($process->getExitCode() === self::SUCCESS) {
    $this->error('All tests passed, try again');

    exit(Command::FAILURE);
    }

    return $this->parseFailingTestClasses($lines);
    }

    /**
    * @param array<string> $lines
    * @return Collection<string>
    */
    protected function parseFailingTestClasses(array $lines): Collection
    {
    return collect($lines)
    ->filter(fn ($line) => Str::startsWith($line, 'Test \''))
    ->filter(fn ($line) => !Str::endsWith($line, 'started'))
    ->map(fn (string $test) => Str::of($test)
    ->replace('Test \'', '')
    ->replace('\' ended', '')
    ->trim()
    ->explode('::')
    ->first())
    ->unique();
    }

    protected function seed(): string
    {
    if (isset($this->seed)) {
    return $this->seed;
    }

    $seed = $this->argument('seed');

    if (! is_string($seed)) {
    $this->error('Seed must be a string');

    $this->exit(Command::FAILURE);
    }

    return $this->seed = $seed;
    }

    protected function phpUnitPath(): string
    {
    return __DIR__ . '/../../../vendor/bin/phpunit';
    }
    }