Last active
July 10, 2022 12:00
-
-
Save CertainLach/5770d7ad4836066f8e0bd91e823e451b to your computer and use it in GitHub Desktop.
Revisions
-
CertainLach revised this gist
Aug 23, 2020 . 1 changed file with 140 additions and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,140 @@ # Nix flake, which generates document above { description = "Jrsonnet benchmark"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; inputs.flake-utils.url = "github:numtide/flake-utils"; inputs.jrsonnet.url = "github:CertainLach/jrsonnet"; outputs = { self, nixpkgs, flake-utils, jrsonnet }: flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; in { defaultPackage = pkgs.stdenv.mkDerivation { name = "jrsonnet-benchmark"; buildInputs = [ jrsonnet.outputs.defaultPackage.${system} pkgs.rustfmt pkgs.cargo pkgs.hyperfine pkgs.jsonnet (pkgs.callPackage pkgs.stdenv.mkDerivation { name = "sjsonnet"; src = pkgs.fetchurl { url = "https://github.com/databricks/sjsonnet/releases/download/0.2.6/sjsonnet.jar"; sha256 = "076fgpj1kmqix157c8cqz617z6r99nwmy125p7gpi8nnrxc90bvs"; }; unpackPhase = "true"; buildInputs = [ pkgs.adoptopenjdk-hotspot-bin-8 pkgs.makeWrapper ]; installPhase = '' mkdir -p $out/bin $out/lib cp $src $out/lib/sjsonner.jar makeWrapper ${pkgs.adoptopenjdk-hotspot-bin-8}/bin/java $out/bin/sjsonnet --add-flags "-jar $out/lib/sjsonner.jar" ''; separateDebugInfo = false; }) (pkgs.callPackage pkgs.stdenv.mkDerivation { name = "gojsonnet"; unpackPhase = "true"; buildInputs = [ pkgs.go-jsonnet ]; installPhase = '' mkdir -p $out/bin echo "Test" cp ${pkgs.go-jsonnet}/bin/jsonnet $out/bin/gojsonnet ''; separateDebugInfo = false; }) ]; src = pkgs.fetchFromGitHub { owner = "google"; repo = "jsonnet"; rev = "2991580ddae4027384502cfe11a31bd032d58247"; sha256 = "15csk6bhvjkjy3fg3g06pg5pds41lgfrgnxr435s8cyy6m1bjr23"; }; buildPhase = "true"; installPhase = let bench = name: { go ? true, scala ? true, cpp ? true, large ? false , comment ? null }: let path = "${name}.jsonnet"; in '' hyperfine --style basic --export-markdown 'result.md' 'jrsonnet ${path}' ${ if go then "'gojsonnet ${if large then "-s 200000" else ""} ${path}'" else "" } ${ if cpp then "'jsonnet ${if large then "-s 200000" else ""} ${path}'" else "" } ${if scala then "'sjsonnet ${path}'" else ""} echo '## ${name}' >> $out ${if comment != null then '' echo >> $out echo "${comment}" >> $out '' else '' echo >> $out ''} cat result.md >> $out echo >> $out ''; in '' echo "# Benchmark results" > $out echo "" >> $out cd perf_tests ${bench "large_string_join" { }} ${bench "large_string_template" { go = false; comment = '' > Golang overflows OS stack on this test ''; }} ${bench "realistic1" { }} ${bench "realistic2" { }} cd ../benchmarks jrsonnet -S gen_big_object.jsonnet > bench.05.gen.jsonnet ${bench "bench.01" { scala = false; comment = '' > Scala impl fails with obscure exception in this test ''; }} ${bench "bench.02" { }} ${bench "bench.03" { }} ${bench "bench.04" { }} ${bench "bench.05.gen" { }} ${bench "bench.06" { scala = false; comment = '' > std.reverse is not implemented in scala ''; }} ${bench "bench.07" { large = true; scala = false; comment = '' > C++ and golang impl overflows stack in this test with default settings > Scala impl fails with stack overflow in this test ''; }} ${bench "bench.08" { }} ''; }; }); } -
Aug 23, 2020 . 1 changed file with 66 additions and 60 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,109 +1,115 @@ # Benchmark results ## large_string_join | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet large_string_join.jsonnet` | 16.5 ± 0.5 | 16.1 | 19.6 | 1.00 | | `gojsonnet large_string_join.jsonnet` | 90.7 ± 3.8 | 85.6 | 104.0 | 5.49 ± 0.29 | | `jsonnet large_string_join.jsonnet` | 69.5 ± 2.4 | 68.2 | 78.1 | 4.20 ± 0.20 | | `sjsonnet large_string_join.jsonnet` | 760.6 ± 12.7 | 739.9 | 787.2 | 46.00 ± 1.67 | ## large_string_template > Golang overflows OS stack on this test | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet large_string_template.jsonnet` | 7.8 ± 0.2 | 7.6 | 10.0 | 1.00 | | `jsonnet large_string_template.jsonnet` | 18415.0 ± 39.4 | 18371.7 | 18478.6 | 2345.88 ± 74.55 | | `sjsonnet large_string_template.jsonnet` | 612.5 ± 7.7 | 602.6 | 628.2 | 78.03 ± 2.66 | ## realistic1 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet realistic1.jsonnet` | 47.1 ± 0.7 | 46.5 | 49.7 | 1.00 | | `gojsonnet realistic1.jsonnet` | 8256.5 ± 41.0 | 8208.7 | 8333.1 | 175.25 ± 2.68 | | `jsonnet realistic1.jsonnet` | 25659.5 ± 229.7 | 25434.8 | 26115.0 | 544.64 ± 9.26 | | `sjsonnet realistic1.jsonnet` | 837.8 ± 18.8 | 809.6 | 866.0 | 17.78 ± 0.47 | ## realistic2 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet realistic2.jsonnet` | 476.6 ± 5.0 | 472.5 | 489.8 | 1.00 | | `gojsonnet realistic2.jsonnet` | 6890.0 ± 112.4 | 6808.3 | 7198.1 | 14.46 ± 0.28 | | `jsonnet realistic2.jsonnet` | 31598.2 ± 645.3 | 30772.8 | 33067.6 | 66.30 ± 1.52 | | `sjsonnet realistic2.jsonnet` | 1997.3 ± 48.1 | 1917.2 | 2053.5 | 4.19 ± 0.11 | ## bench.01 > Scala impl fails with obscure exception in this test | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.01.jsonnet` | 1.7 ± 0.3 | 1.5 | 3.4 | 1.00 | | `gojsonnet bench.01.jsonnet` | 4.6 ± 0.4 | 4.3 | 6.5 | 2.64 ± 0.46 | | `jsonnet bench.01.jsonnet` | 16.4 ± 1.3 | 15.8 | 27.7 | 9.37 ± 1.64 | ## bench.02 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.02.jsonnet` | 1.4 ± 0.1 | 1.3 | 2.4 | 1.00 | | `gojsonnet bench.02.jsonnet` | 984.9 ± 23.1 | 960.0 | 1021.1 | 715.07 ± 46.26 | | `jsonnet bench.02.jsonnet` | 2673.7 ± 93.9 | 2550.2 | 2836.6 | 1941.17 ± 135.48 | | `sjsonnet bench.02.jsonnet` | 851.7 ± 48.1 | 784.8 | 931.2 | 618.34 ± 51.10 | ## bench.03 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.03.jsonnet` | 127.4 ± 3.2 | 121.7 | 132.1 | 1.00 | | `gojsonnet bench.03.jsonnet` | 394.7 ± 14.8 | 374.9 | 420.2 | 3.10 ± 0.14 | | `jsonnet bench.03.jsonnet` | 163.0 ± 6.9 | 154.8 | 175.1 | 1.28 ± 0.06 | | `sjsonnet bench.03.jsonnet` | 891.3 ± 33.5 | 843.8 | 951.1 | 7.00 ± 0.31 | ## bench.04 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.04.jsonnet` | 173.7 ± 6.9 | 162.4 | 182.8 | 1.00 | | `gojsonnet bench.04.jsonnet` | 417.7 ± 21.2 | 390.0 | 454.3 | 2.40 ± 0.16 | | `jsonnet bench.04.jsonnet` | 1253.5 ± 54.0 | 1205.7 | 1348.8 | 7.22 ± 0.42 | | `sjsonnet bench.04.jsonnet` | 1223.4 ± 147.7 | 1014.8 | 1458.6 | 7.04 ± 0.90 | ## bench.05.gen | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.05.gen.jsonnet` | 690.3 ± 21.5 | 660.8 | 736.7 | 1.00 | | `gojsonnet bench.05.gen.jsonnet` | 1367.8 ± 22.2 | 1337.9 | 1395.6 | 1.98 ± 0.07 | | `jsonnet bench.05.gen.jsonnet` | 1880.8 ± 42.2 | 1823.0 | 1931.6 | 2.72 ± 0.10 | | `sjsonnet bench.05.gen.jsonnet` | 868.7 ± 29.8 | 817.7 | 907.6 | 1.26 ± 0.06 | ## bench.06 > std.reverse is not implemented in scala | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.06.jsonnet` | 3.6 ± 0.4 | 3.1 | 5.6 | 1.00 | | `gojsonnet bench.06.jsonnet` | 12.8 ± 1.2 | 11.1 | 18.9 | 3.58 ± 0.49 | | `jsonnet bench.06.jsonnet` | 3951.9 ± 148.4 | 3799.0 | 4214.1 | 1109.22 ± 121.41 | ## bench.07 > C++ and golang impl overflows stack in this test with default settings > Scala impl fails with stack overflow in this test | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.07.jsonnet` | 1.7 ± 0.1 | 1.6 | 2.8 | 1.00 | | `gojsonnet -s 200000 bench.07.jsonnet` | 715.1 ± 14.6 | 697.9 | 734.6 | 423.13 ± 30.66 | | `jsonnet -s 200000 bench.07.jsonnet` | 167.1 ± 2.9 | 165.1 | 175.6 | 98.89 ± 7.08 | ## bench.08 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.08.jsonnet` | 1.3 ± 0.1 | 1.2 | 2.3 | 1.00 | | `gojsonnet bench.08.jsonnet` | 1.6 ± 0.2 | 1.4 | 2.8 | 1.17 ± 0.18 | | `jsonnet bench.08.jsonnet` | 1309.8 ± 40.9 | 1282.4 | 1415.8 | 971.10 ± 80.36 | | `sjsonnet bench.08.jsonnet` | 498.5 ± 10.5 | 485.8 | 514.1 | 369.56 ± 29.37 | -
Jul 20, 2020 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,109 @@ # Benchmark results Last updated 21/06/2020, jrsonnet wins everywhere except bench.03.jsonnet, where it loses only to C++ impl, because it has better function stack management. ## bench.01.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.01.jsonnet` | 3.5 ± 0.6 | 2.6 | 6.9 | 1.00 | | `gojsonnet bench.01.jsonnet` | 8.0 ± 0.8 | 6.8 | 11.2 | 2.28 ± 0.43 | | `jsonnet bench.01.jsonnet` | 19.1 ± 2.2 | 17.5 | 37.0 | 5.43 ± 1.06 | | `sjsonnet bench.01.jsonnet` | 546.3 ± 13.0 | 525.7 | 575.7 | 155.38 ± 24.93 | ## bench.02.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.02.jsonnet` | 2.0 ± 0.5 | 1.3 | 3.9 | 1.00 | | `gojsonnet bench.02.jsonnet` | 966.8 ± 15.6 | 931.7 | 989.3 | 484.68 ± 111.08 | | `jsonnet bench.02.jsonnet` | 2840.6 ± 81.0 | 2776.5 | 3056.4 | 1424.02 ± 328.06 | | `sjsonnet bench.02.jsonnet` | 872.3 ± 22.9 | 841.5 | 923.5 | 437.28 ± 100.62 | ## bench.03.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.03.jsonnet` | 239.8 ± 3.3 | 236.6 | 247.0 | 1.36 ± 0.08 | | `gojsonnet bench.03.jsonnet` | 380.8 ± 4.0 | 375.0 | 388.8 | 2.16 ± 0.13 | | `jsonnet bench.03.jsonnet` | 176.1 ± 10.2 | 170.7 | 210.0 | 1.00 | | `sjsonnet bench.03.jsonnet` | 961.0 ± 37.6 | 919.8 | 1056.3 | 5.46 ± 0.38 | ## bench.04.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.04.jsonnet` | 146.1 ± 3.6 | 142.3 | 156.5 | 1.00 | | `gojsonnet bench.04.jsonnet` | 378.1 ± 8.5 | 366.5 | 396.0 | 2.59 ± 0.09 | | `jsonnet bench.04.jsonnet` | 1333.3 ± 14.5 | 1313.7 | 1364.8 | 9.12 ± 0.24 | | `sjsonnet bench.04.jsonnet` | 1232.0 ± 83.4 | 1104.9 | 1329.9 | 8.43 ± 0.61 | ## bench.06.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.06.jsonnet` | 4.4 ± 0.5 | 3.4 | 6.9 | 1.00 | | `gojsonnet bench.06.jsonnet` | 16.0 ± 1.2 | 14.0 | 21.0 | 3.61 ± 0.52 | | `jsonnet bench.06.jsonnet` | 4531.0 ± 75.9 | 4459.6 | 4685.5 | 1025.72 ± 124.25 | ## bench.07.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.07.jsonnet` | 2.5 ± 0.5 | 1.7 | 4.8 | 1.00 | | `gojsonnet -s 200000 bench.07.jsonnet` | 578.9 ± 22.1 | 549.6 | 625.7 | 236.04 ± 44.86 | | `jsonnet -s 200000 bench.07.jsonnet` | 207.4 ± 20.0 | 192.9 | 272.2 | 84.55 ± 17.73 | | `sjsonnet bench.07.jsonnet` | 1140.3 ± 93.9 | 989.2 | 1303.8 | 464.98 ± 94.66 | ## bench.08.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.08.jsonnet` | 1.9 ± 0.6 | 1.1 | 4.6 | 1.00 | | `gojsonnet bench.08.jsonnet` | 4.1 ± 0.6 | 3.2 | 6.1 | 2.17 ± 0.76 | | `jsonnet bench.08.jsonnet` | 1397.2 ± 101.2 | 1340.4 | 1674.4 | 735.73 ± 243.36 | | `sjsonnet bench.08.jsonnet` | 526.1 ± 22.8 | 506.3 | 576.0 | 277.04 ± 90.22 | ## bench.05.gen.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet bench.05.gen.jsonnet` | 561.9 ± 6.2 | 553.6 | 575.3 | 1.00 | | `gojsonnet bench.05.gen.jsonnet` | 1328.2 ± 10.0 | 1315.7 | 1340.6 | 2.36 ± 0.03 | | `jsonnet bench.05.gen.jsonnet` | 2197.9 ± 22.4 | 2160.5 | 2233.2 | 3.91 ± 0.06 | | `sjsonnet bench.05.gen.jsonnet` | 848.0 ± 11.6 | 828.3 | 863.3 | 1.51 ± 0.03 | ## large_string_join.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet large_string_join.jsonnet` | 21.2 ± 1.0 | 19.7 | 25.7 | 1.00 | | `gojsonnet large_string_join.jsonnet` | 100.7 ± 3.1 | 96.5 | 108.0 | 4.75 ± 0.28 | | `jsonnet large_string_join.jsonnet` | 79.8 ± 1.2 | 77.8 | 82.8 | 3.76 ± 0.19 | | `sjsonnet large_string_join.jsonnet` | 839.4 ± 25.5 | 802.9 | 888.7 | 39.57 ± 2.30 | ## large_string_template.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet large_string_template.jsonnet` | 9.8 ± 0.9 | 8.8 | 16.3 | 1.00 | | `jsonnet large_string_template.jsonnet` | 20877.0 ± 460.1 | 20571.4 | 22109.6 | 2140.01 ± 206.15 | | `sjsonnet large_string_template.jsonnet` | 665.8 ± 26.0 | 639.6 | 727.6 | 68.25 ± 6.93 | ## realistic1.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet realistic1.jsonnet` | 74.1 ± 2.7 | 72.0 | 89.0 | 1.00 | | `gojsonnet realistic1.jsonnet` | 8462.7 ± 373.0 | 8138.9 | 9295.2 | 114.25 ± 6.53 | | `jsonnet realistic1.jsonnet` | 28181.6 ± 726.0 | 27396.6 | 29514.3 | 380.47 ± 16.98 | | `sjsonnet realistic1.jsonnet` | 937.0 ± 36.8 | 881.2 | 987.9 | 12.65 ± 0.68 | ## realistic2.jsonnet | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `jrsonnet realistic2.jsonnet` | 4.6 ± 0.6 | 3.7 | 8.0 | 1.00 | | `gojsonnet realistic2.jsonnet` | 7622.5 ± 534.8 | 7254.5 | 8904.7 | 1655.50 ± 251.88 | | `jsonnet realistic2.jsonnet` | 37129.3 ± 1492.2 | 34479.6 | 39605.1 | 8064.00 ± 1135.93 | | `sjsonnet realistic2.jsonnet` | 2154.1 ± 46.8 | 2091.7 | 2237.0 | 467.84 ± 63.98 |