Skip to content

Instantly share code, notes, and snippets.

@mszabo-wikia

mszabo-wikia/ExcimerPyroscopeIntegration.php

Last active February 4, 2024 14:43
Show Gist options
  • Save mszabo-wikia/1ddee8e6d896a55cd3ef92f88eb93b0c to your computer and use it in GitHub Desktop.
Save mszabo-wikia/1ddee8e6d896a55cd3ef92f88eb93b0c to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. mszabo-wikia revised this gist Feb 4, 2024. 1 changed file with 15 additions and 0 deletions.
    15 changes: 15 additions & 0 deletions ExcimerPyroscopeIntegration.php
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,19 @@
    <?php
    /**
    * Copyright 2024 Máté Szabó
    *
    * Licensed under the Apache License, Version 2.0 (the "License");
    * you may not use this file except in compliance with the License.
    * You may obtain a copy of the License at
    *
    * https://www.apache.org/licenses/LICENSE-2.0
    *
    * Unless required by applicable law or agreed to in writing, software
    * distributed under the License is distributed on an "AS IS" BASIS,
    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    * See the License for the specific language governing permissions and
    * limitations under the License.
    */

    use Perftools\Profiles\Line;
    use Perftools\Profiles\Location;
  2. mszabo-wikia created this gist Feb 4, 2024.
    239 changes: 239 additions & 0 deletions ExcimerPyroscopeIntegration.php
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,239 @@
    <?php

    use Perftools\Profiles\Line;
    use Perftools\Profiles\Location;
    use Perftools\Profiles\PBFunction;
    use Perftools\Profiles\Profile;
    use Perftools\Profiles\Sample;
    use Perftools\Profiles\ValueType;

    /**
    * Proof of concept for integrating <a href="https://www.mediawiki.org/wiki/Excimer">Excimer</a>
    * profiling with Pyroscope.
    * It is best to have PHP automatically load this file before execution via the `auto_prepend_file` INI directive,
    * like so:
    * ```
    * $ php -dauto_prepend_file=/path/to/ExcimerPyroscopeIntegration.php /path/to/entry/point.php
    * ```
    *
    * This class configures an Excimer profiler to sample PHP stack traces and converts the collected
    * data into a pprof profile, which is then sent to Pyroscope.
    *
    * The Excimer and cURL extensions are required for this to work, alongside with either the protobuf extension
    * or the google/protobuf Composer package.
    * You will also need to generate the PHP classes for the pprof profile format using the `protoc` tool
    * and place them somewhere where your application can autoload them.
    */
    class ExcimerPyroscopeIntegration {
    /**
    * Excimer profiler instance - this needs to be kept in scope until the end of the request.
    * @var ExcimerProfiler
    */
    private static ExcimerProfiler $profiler;

    /**
    * The Pyroscope server to send the profile to.
    * @var string
    */
    private static string $pyroscopeHost;

    /**
    * The name of the application to profile.
    * @var string
    */
    private static string $appName;

    /**
    * Whether to output the stack traces in stack-collapsed format to a file.
    * This is useful for generating flamegraphs via Brendan Gregg's scripts to compare the output
    * with the one generated by Pyroscope.
    * @var bool
    */
    private static bool $outputCollapsedStacks;

    /**
    * The string table for the pprof profile.
    * @var string[]
    */
    private static array $stringTable = [ '' ];

    /**
    * Initialize the profiler and register a shutdown function to send the profile to Pyroscope.
    * @param array $config Configuration options:
    * - pyroscopeHost: The Pyroscope server to send the profile to.
    * - samplingPeriod: The sampling period for the profiler.
    * - outputCollapsedStacks: Whether to output the stack traces in stack-collapsed format to a file (default: false)
    * @return void
    */
    public static function init( array $config ): void {
    self::$pyroscopeHost = $config['pyroscopeHost'];
    self::$appName = $config['appName'];
    self::$outputCollapsedStacks = $config['outputCollapsedStacks'] ?? false;

    self::$profiler = new ExcimerProfiler();
    self::$profiler->setPeriod( $config['samplingPeriod'] );
    self::$profiler->setEventType( EXCIMER_REAL );
    self::$profiler->start();

    register_shutdown_function( [ self::class, 'shutdown' ] );
    }

    /**
    * Stop the profiler and send collected data to Pyroscope.
    * @return void
    */
    private static function shutdown(): void {
    self::$profiler->stop();

    $log = self::$profiler->getLog();

    // Location and function IDs to be used in this pprof profile.
    $nextLocationId = 1;
    $nextFunctionId = 1;

    // List of pprof samples.
    $samples = [];
    // Map of pprof function objects keyed by identifier.
    $functionsByIdentifier = [];
    // Map of pprof location objects keyed by identifier.
    $locationsByIdentifier = [];

    // UNIX timestamp in seconds denoting the start of the profile
    $firstTimestamp = null;
    // UNIX timestamp in seconds denoting the end of the profile
    $lastTimestamp = null;

    /** @var ExcimerLogEntry $entry */
    foreach ( $log as $entry ) {
    // Excimer timestamps are relative, so backdate the start of profiling
    // to the time PHP started processing this request.
    $firstTimestamp ??= $_SERVER['REQUEST_TIME_FLOAT'] + $entry->getTimestamp();
    $lastTimestamp = $entry->getTimestamp();

    // Walk the stack trace of this log entry and determine the pprof location ID for each frame
    // for inclusion in a new sample.
    $trace = $entry->getTrace();
    $locationIds = [];
    foreach ( $trace as $frame ) {
    // Determine the identifier of this stack frame.
    // For class methods, we can trivially use the class + method name,
    // while for global function calls or file-scope code,
    // we use the function name and the file name, respectively.
    if ( isset( $frame['function'] ) ) {
    if ( isset( $frame['class'] ) ) {
    $identifier = $frame['class'] . '::' . $frame['function'];
    } else {
    $identifier = $frame['function'];
    }
    } else {
    $identifier = $frame['file'];
    }

    // Mark this identifier in the pprof function and location tables
    // if not seen yet.
    if ( !isset( $functionsByIdentifier[$identifier] ) ) {
    $functionNameIndex = self::addToStringTable( $identifier );
    $fileNameIndex = self::addToStringTable( $frame['file'] );

    $functionsByIdentifier[$identifier] = new PBFunction( [
    'id' => $nextFunctionId++,
    'name' => $functionNameIndex,
    'filename' => $fileNameIndex,
    'start_line' => $frame['line']
    ] );

    $locationsByIdentifier[$identifier] = new Location( [
    'id' => $nextLocationId++,
    'line' => [
    new Line( [
    'function_id' => $functionsByIdentifier[$identifier]->getId(),
    'line' => $frame['line'],
    ] ),
    ]
    ] );
    }

    $locationIds[] = $locationsByIdentifier[$identifier]->getId();
    }

    $samples[] = new Sample( [
    'location_id' => $locationIds,
    'value' => [ $entry->getEventCount() ],
    ] );
    }

    $wallIndex = self::addToStringTable( 'wall' );
    $secondsIndex = self::addToStringTable( 'seconds' );

    // Convert the start and end timestamps to nanoseconds, as expected by pprof and pyroscope.
    $firstTimestamp = (int)( 1e9 * $firstTimestamp );
    $lastTimestamp = (int)( 1e9 * $lastTimestamp );

    $profile = new Profile( [
    'sample_type' => [ new ValueType( [ 'type' => $wallIndex, 'unit' => $secondsIndex ] ) ],
    'sample' => $samples,
    'location' => array_values( $locationsByIdentifier ),
    'function' => array_values( $functionsByIdentifier ),
    'time_nanos' => $firstTimestamp,
    'duration_nanos' => $lastTimestamp - $firstTimestamp,
    'string_table' => self::$stringTable,
    ] );

    $query = http_build_query( [
    'name' => self::$appName,
    'format' => 'pprof',
    'from' => $firstTimestamp,
    'to' => $lastTimestamp,
    ] );

    $curl = curl_init();
    curl_setopt_array( $curl, [
    CURLOPT_HEADER => 1,
    CURLOPT_URL => self::$pyroscopeHost . 'ingest?' . $query,
    CURLOPT_POST => true,
    CURLOPT_POSTFIELDS => $profile->serializeToString(),
    CURLOPT_HTTPHEADER => [
    'Content-Type: application/octet-stream'
    ],
    CURLOPT_RETURNTRANSFER => true
    ] );

    $result = curl_exec( $curl );
    if ( $result === false ) {
    error_log( 'Failed to send profile to Pyroscope: ' . curl_error( $curl ) );
    } else {
    $info = curl_getinfo( $curl );
    if ( $info['http_code'] !== 200 ) {
    error_log( 'Error while sending profile to Pyroscope: ' . $result );
    }
    }

    // Record the profile to disk in stack-collapsed format as well
    // so that it could be processed by Brendan Gregg's scripts for comparison to Pyroscope.
    if ( self::$outputCollapsedStacks ) {
    $collapsed = $log->formatCollapsed();

    file_put_contents( '/app/profile/excimer.log', $collapsed );
    }
    }

    /**
    * Add a string to the string table that will be used for this pprof profile and return its index.
    * The string is not deduplicated; calling this function with the same string will add it to the table again.
    *
    * @param string $string The string to add to the table
    * @return int Index of the string in the string table
    */
    private static function addToStringTable( string $string ): int {
    $id = count( self::$stringTable );
    self::$stringTable[] = $string;
    return $id;
    }
    }

    ExcimerPyroscopeIntegration::init( [
    'pyroscopeHost' => 'http://host.docker.internal:9090/',
    'appName' => 'php-test',
    'samplingPeriod' => 0.001,
    'outputCollapsedStacks' => true,
    ] );