mszabo-wikia · February 4, 2024 14:43
diff --git a/ExcimerPyroscopeIntegration.php b/ExcimerPyroscopeIntegration.php
 <?php
 /**
 * Copyright 2024 Máté Szabó
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

 use Perftools\Profiles\Line;
 use Perftools\Profiles\Location;
 use Perftools\Profiles\PBFunction;
 use Perftools\Profiles\Profile;
 use Perftools\Profiles\Sample;
 use Perftools\Profiles\ValueType;

 /**
 * Proof of concept for integrating <a href="https://www.mediawiki.org/wiki/Excimer">Excimer</a>
 * profiling with Pyroscope.
 * It is best to have PHP automatically load this file before execution via the `auto_prepend_file` INI directive,
 * like so:
 * ```
 * $ php -dauto_prepend_file=/path/to/ExcimerPyroscopeIntegration.php /path/to/entry/point.php
 * ```
 *
 * This class configures an Excimer profiler to sample PHP stack traces and converts the collected
 * data into a pprof profile, which is then sent to Pyroscope.
 *
 * The Excimer and cURL extensions are required for this to work, alongside with either the protobuf extension
 * or the google/protobuf Composer package.
 * You will also need to generate the PHP classes for the pprof profile format using the `protoc` tool
 * and place them somewhere where your application can autoload them.
 */
 class ExcimerPyroscopeIntegration {
 	/**
 	 * Excimer profiler instance - this needs to be kept in scope until the end of the request.
 	 * @var ExcimerProfiler
 	 */
 	private static ExcimerProfiler $profiler;

 	/**
 	 * The Pyroscope server to send the profile to.
 	 * @var string
 	 */
 	private static string $pyroscopeHost;

 	/**
 	 * The name of the application to profile.
 	 * @var string
 	 */
 	private static string $appName;

 	/**
 	 * Whether to output the stack traces in stack-collapsed format to a file.
 	 * This is useful for generating flamegraphs via Brendan Gregg's scripts to compare the output
 	 * with the one generated by Pyroscope.
 	 * @var bool
 	 */
 	private static bool $outputCollapsedStacks;

 	/**
 	 * The string table for the pprof profile.
 	 * @var string[]
 	 */
 	private static array $stringTable = [ '' ];

 	/**
 	 * Initialize the profiler and register a shutdown function to send the profile to Pyroscope.
 	 * @param array $config Configuration options:
 	 *  - pyroscopeHost: The Pyroscope server to send the profile to.
 	 *  - samplingPeriod: The sampling period for the profiler.
 	 *  - outputCollapsedStacks: Whether to output the stack traces in stack-collapsed format to a file (default: false)
 	 * @return void
 	 */
 	public static function init( array $config ): void {
 		self::$pyroscopeHost = $config['pyroscopeHost'];
 		self::$appName = $config['appName'];
 		self::$outputCollapsedStacks = $config['outputCollapsedStacks'] ?? false;

 		self::$profiler = new ExcimerProfiler();
 		self::$profiler->setPeriod( $config['samplingPeriod'] );
 		self::$profiler->setEventType( EXCIMER_REAL );
 		self::$profiler->start();

 		register_shutdown_function( [ self::class, 'shutdown' ] );
 	}

 	/**
 	 * Stop the profiler and send collected data to Pyroscope.
 	 * @return void
 	 */
 	private static function shutdown(): void {
 		self::$profiler->stop();

 		$log = self::$profiler->getLog();

 		// Location and function IDs to be used in this pprof profile.
 		$nextLocationId = 1;
 		$nextFunctionId = 1;

 		// List of pprof samples.
 		$samples = [];
 		// Map of pprof function objects keyed by identifier.
 		$functionsByIdentifier = [];
 		// Map of pprof location objects keyed by identifier.
 		$locationsByIdentifier = [];

 		// UNIX timestamp in seconds denoting the start of the profile
 		$firstTimestamp = null;
 		// UNIX timestamp in seconds denoting the end of the profile
 		$lastTimestamp = null;

 		/** @var ExcimerLogEntry $entry */
 		foreach ( $log as $entry ) {
 			// Excimer timestamps are relative, so backdate the start of profiling
 			// to the time PHP started processing this request.
 			$firstTimestamp ??= $_SERVER['REQUEST_TIME_FLOAT'] + $entry->getTimestamp();
 			$lastTimestamp = $entry->getTimestamp();

 			// Walk the stack trace of this log entry and determine the pprof location ID for each frame
 			// for inclusion in a new sample.
 			$trace = $entry->getTrace();
 			$locationIds = [];
 			foreach ( $trace as $frame ) {
 				// Determine the identifier of this stack frame.
 				// For class methods, we can trivially use the class + method name,
 				// while for global function calls or file-scope code,
 				// we use the function name and the file name, respectively.
 				if ( isset( $frame['function'] ) ) {
 					if ( isset( $frame['class'] ) ) {
 						$identifier = $frame['class'] . '::' . $frame['function'];
 					} else {
 						$identifier = $frame['function'];
 					}
 				} else {
 					$identifier = $frame['file'];
 				}

 				// Mark this identifier in the pprof function and location tables
 				// if not seen yet.
 				if ( !isset( $functionsByIdentifier[$identifier] ) ) {
 					$functionNameIndex = self::addToStringTable( $identifier );
 					$fileNameIndex = self::addToStringTable( $frame['file'] );

 					$functionsByIdentifier[$identifier] = new PBFunction( [
 						'id' => $nextFunctionId++,
 						'name' => $functionNameIndex,
 						'filename' => $fileNameIndex,
 						'start_line' => $frame['line']
 					] );

 					$locationsByIdentifier[$identifier] = new Location( [
 						'id' => $nextLocationId++,
 						'line' => [
 							new Line( [
 								'function_id' => $functionsByIdentifier[$identifier]->getId(),
 								'line' => $frame['line'],
 							] ),
 						]
 					] );
 				}

 				$locationIds[] = $locationsByIdentifier[$identifier]->getId();
 			}

 			$samples[] = new Sample( [
 				'location_id' => $locationIds,
 				'value' => [ $entry->getEventCount() ],
 			] );
 		}

 		$wallIndex = self::addToStringTable( 'wall' );
 		$secondsIndex = self::addToStringTable( 'seconds' );

 		// Convert the start and end timestamps to nanoseconds, as expected by pprof and pyroscope.
 		$firstTimestamp = (int)( 1e9 * $firstTimestamp );
 		$lastTimestamp = (int)( 1e9 * $lastTimestamp );

 		$profile = new Profile( [
 			'sample_type' => [ new ValueType( [ 'type' => $wallIndex, 'unit' => $secondsIndex ] ) ],
 			'sample' => $samples,
 			'location' => array_values( $locationsByIdentifier ),
 			'function' => array_values( $functionsByIdentifier ),
 			'time_nanos' => $firstTimestamp,
 			'duration_nanos' => $lastTimestamp - $firstTimestamp,
 			'string_table' => self::$stringTable,
 		] );

 		$query = http_build_query( [
 			'name' => self::$appName,
 			'format' => 'pprof',
 			'from' => $firstTimestamp,
 			'to' => $lastTimestamp,
 		] );

 		$curl = curl_init();
 		curl_setopt_array( $curl, [
 			CURLOPT_HEADER => 1,
 			CURLOPT_URL => self::$pyroscopeHost . 'ingest?' . $query,
 			CURLOPT_POST => true,
 			CURLOPT_POSTFIELDS => $profile->serializeToString(),
 			CURLOPT_HTTPHEADER => [
 				'Content-Type: application/octet-stream'
 			],
 			CURLOPT_RETURNTRANSFER => true
 		] );

 		$result = curl_exec( $curl );
 		if ( $result === false ) {
 			error_log( 'Failed to send profile to Pyroscope: ' . curl_error( $curl ) );
 		} else {
 			$info = curl_getinfo( $curl );
 			if ( $info['http_code'] !== 200 ) {
 				error_log( 'Error while sending profile to Pyroscope: ' . $result );
 			}
 		}

 		// Record the profile to disk in stack-collapsed format as well
 		// so that it could be processed by Brendan Gregg's scripts for comparison to Pyroscope.
 		if ( self::$outputCollapsedStacks ) {
 			$collapsed = $log->formatCollapsed();

 			file_put_contents( '/app/profile/excimer.log', $collapsed );
 		}
 	}

 	/**
 	 * Add a string to the string table that will be used for this pprof profile and return its index.
 	 * The string is not deduplicated; calling this function with the same string will add it to the table again.
 	 *
 	 * @param string $string The string to add to the table
 	 * @return int Index of the string in the string table
 	 */
 	private static function addToStringTable( string $string ): int {
 		$id = count( self::$stringTable );
 		self::$stringTable[] = $string;
 		return $id;
 	}
 }

 ExcimerPyroscopeIntegration::init( [
 	'pyroscopeHost' => 'http://host.docker.internal:9090/',
 	'appName' => 'php-test',
 	'samplingPeriod' => 0.001,
 	'outputCollapsedStacks' => true,
 ] );
	<?php
	/**
	* Copyright 2024 Máté Szabó
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* https://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	use Perftools\Profiles\Line;
	use Perftools\Profiles\Location;
	use Perftools\Profiles\PBFunction;
	use Perftools\Profiles\Profile;
	use Perftools\Profiles\Sample;
	use Perftools\Profiles\ValueType;

	/**
	* Proof of concept for integrating <a href="https://www.mediawiki.org/wiki/Excimer">Excimer</a>
	* profiling with Pyroscope.
	* It is best to have PHP automatically load this file before execution via the `auto_prepend_file` INI directive,
	* like so:
	* ```
	* $ php -dauto_prepend_file=/path/to/ExcimerPyroscopeIntegration.php /path/to/entry/point.php
	* ```
	*
	* This class configures an Excimer profiler to sample PHP stack traces and converts the collected
	* data into a pprof profile, which is then sent to Pyroscope.
	*
	* The Excimer and cURL extensions are required for this to work, alongside with either the protobuf extension
	* or the google/protobuf Composer package.
	* You will also need to generate the PHP classes for the pprof profile format using the `protoc` tool
	* and place them somewhere where your application can autoload them.
	*/
	class ExcimerPyroscopeIntegration {
	/**
	* Excimer profiler instance - this needs to be kept in scope until the end of the request.
	* @var ExcimerProfiler
	*/
	private static ExcimerProfiler $profiler;

	/**
	* The Pyroscope server to send the profile to.
	* @var string
	*/
	private static string $pyroscopeHost;

	/**
	* The name of the application to profile.
	* @var string
	*/
	private static string $appName;

	/**
	* Whether to output the stack traces in stack-collapsed format to a file.
	* This is useful for generating flamegraphs via Brendan Gregg's scripts to compare the output
	* with the one generated by Pyroscope.
	* @var bool
	*/
	private static bool $outputCollapsedStacks;

	/**
	* The string table for the pprof profile.
	* @var string[]
	*/
	private static array $stringTable = [ '' ];

	/**
	* Initialize the profiler and register a shutdown function to send the profile to Pyroscope.
	* @param array $config Configuration options:
	* - pyroscopeHost: The Pyroscope server to send the profile to.
	* - samplingPeriod: The sampling period for the profiler.
	* - outputCollapsedStacks: Whether to output the stack traces in stack-collapsed format to a file (default: false)
	* @return void
	*/
	public static function init( array $config ): void {
	self::$pyroscopeHost = $config['pyroscopeHost'];
	self::$appName = $config['appName'];
	self::$outputCollapsedStacks = $config['outputCollapsedStacks'] ?? false;

	self::$profiler = new ExcimerProfiler();
	self::$profiler->setPeriod( $config['samplingPeriod'] );
	self::$profiler->setEventType( EXCIMER_REAL );
	self::$profiler->start();

	register_shutdown_function( [ self::class, 'shutdown' ] );
	}

	/**
	* Stop the profiler and send collected data to Pyroscope.
	* @return void
	*/
	private static function shutdown(): void {
	self::$profiler->stop();

	$log = self::$profiler->getLog();

	// Location and function IDs to be used in this pprof profile.
	$nextLocationId = 1;
	$nextFunctionId = 1;

	// List of pprof samples.
	$samples = [];
	// Map of pprof function objects keyed by identifier.
	$functionsByIdentifier = [];
	// Map of pprof location objects keyed by identifier.
	$locationsByIdentifier = [];

	// UNIX timestamp in seconds denoting the start of the profile
	$firstTimestamp = null;
	// UNIX timestamp in seconds denoting the end of the profile
	$lastTimestamp = null;

	/** @var ExcimerLogEntry $entry */
	foreach ( $log as $entry ) {
	// Excimer timestamps are relative, so backdate the start of profiling
	// to the time PHP started processing this request.
	$firstTimestamp ??= $_SERVER['REQUEST_TIME_FLOAT'] + $entry->getTimestamp();
	$lastTimestamp = $entry->getTimestamp();

	// Walk the stack trace of this log entry and determine the pprof location ID for each frame
	// for inclusion in a new sample.
	$trace = $entry->getTrace();
	$locationIds = [];
	foreach ( $trace as $frame ) {
	// Determine the identifier of this stack frame.
	// For class methods, we can trivially use the class + method name,
	// while for global function calls or file-scope code,
	// we use the function name and the file name, respectively.
	if ( isset( $frame['function'] ) ) {
	if ( isset( $frame['class'] ) ) {
	$identifier = $frame['class'] . '::' . $frame['function'];
	} else {
	$identifier = $frame['function'];
	}
	} else {
	$identifier = $frame['file'];
	}

	// Mark this identifier in the pprof function and location tables
	// if not seen yet.
	if ( !isset( $functionsByIdentifier[$identifier] ) ) {
	$functionNameIndex = self::addToStringTable( $identifier );
	$fileNameIndex = self::addToStringTable( $frame['file'] );

	$functionsByIdentifier[$identifier] = new PBFunction( [
	'id' => $nextFunctionId++,
	'name' => $functionNameIndex,
	'filename' => $fileNameIndex,
	'start_line' => $frame['line']
	] );

	$locationsByIdentifier[$identifier] = new Location( [
	'id' => $nextLocationId++,
	'line' => [
	new Line( [
	'function_id' => $functionsByIdentifier[$identifier]->getId(),
	'line' => $frame['line'],
	] ),
	]
	] );
	}

	$locationIds[] = $locationsByIdentifier[$identifier]->getId();
	}

	$samples[] = new Sample( [
	'location_id' => $locationIds,
	'value' => [ $entry->getEventCount() ],
	] );
	}

	$wallIndex = self::addToStringTable( 'wall' );
	$secondsIndex = self::addToStringTable( 'seconds' );

	// Convert the start and end timestamps to nanoseconds, as expected by pprof and pyroscope.
	$firstTimestamp = (int)( 1e9 * $firstTimestamp );
	$lastTimestamp = (int)( 1e9 * $lastTimestamp );

	$profile = new Profile( [
	'sample_type' => [ new ValueType( [ 'type' => $wallIndex, 'unit' => $secondsIndex ] ) ],
	'sample' => $samples,
	'location' => array_values( $locationsByIdentifier ),
	'function' => array_values( $functionsByIdentifier ),
	'time_nanos' => $firstTimestamp,
	'duration_nanos' => $lastTimestamp - $firstTimestamp,
	'string_table' => self::$stringTable,
	] );

	$query = http_build_query( [
	'name' => self::$appName,
	'format' => 'pprof',
	'from' => $firstTimestamp,
	'to' => $lastTimestamp,
	] );

	$curl = curl_init();
	curl_setopt_array( $curl, [
	CURLOPT_HEADER => 1,
	CURLOPT_URL => self::$pyroscopeHost . 'ingest?' . $query,
	CURLOPT_POST => true,
	CURLOPT_POSTFIELDS => $profile->serializeToString(),
	CURLOPT_HTTPHEADER => [
	'Content-Type: application/octet-stream'
	],
	CURLOPT_RETURNTRANSFER => true
	] );

	$result = curl_exec( $curl );
	if ( $result === false ) {
	error_log( 'Failed to send profile to Pyroscope: ' . curl_error( $curl ) );
	} else {
	$info = curl_getinfo( $curl );
	if ( $info['http_code'] !== 200 ) {
	error_log( 'Error while sending profile to Pyroscope: ' . $result );
	}
	}

	// Record the profile to disk in stack-collapsed format as well
	// so that it could be processed by Brendan Gregg's scripts for comparison to Pyroscope.
	if ( self::$outputCollapsedStacks ) {
	$collapsed = $log->formatCollapsed();

	file_put_contents( '/app/profile/excimer.log', $collapsed );
	}
	}

	/**
	* Add a string to the string table that will be used for this pprof profile and return its index.
	* The string is not deduplicated; calling this function with the same string will add it to the table again.
	*
	* @param string $string The string to add to the table
	* @return int Index of the string in the string table
	*/
	private static function addToStringTable( string $string ): int {
	$id = count( self::$stringTable );
	self::$stringTable[] = $string;
	return $id;
	}
	}

	ExcimerPyroscopeIntegration::init( [
	'pyroscopeHost' => 'http://host.docker.internal:9090/',
	'appName' => 'php-test',
	'samplingPeriod' => 0.001,
	'outputCollapsedStacks' => true,
	] );