Skip to content

Instantly share code, notes, and snippets.

@benweint

benweint/build-rubies.rb

Created December 23, 2013 17:49
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save benweint/8101511 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save benweint/8101511 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. benweint created this gist Dec 23, 2013.
    139 changes: 139 additions & 0 deletions build-rubies.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,139 @@
    #!/usr/bin/env ruby

    #
    # This is a proof-of-concept script to build every commit of ruby over the past
    # year and store all of the resulting builds in git. The purpose of using git
    # for storage is to take advantage of its de-duplication and compression
    # features.
    #
    # Requirements:
    # - git
    # - local clone of https://github.com/ruby/ruby
    # - ruby-build: https://github.com/sstephenson/ruby-build
    # - rugged gem: https://github.com/libgit2/rugged
    #
    # This script will build each commit of Ruby, starting with the HEAD commit of
    # your local clone and going back 1 year.
    #
    # Built Rubies will be stored in a git repository, where each commit maps to one
    # source commit. Built artifacts will be stored in a directory called 'work'
    # under the CWD. work/ruby-builds will contain the git repository with the
    # builds.
    #
    # To get a reliable estimate of the space taken up by all of the builds, be sure
    # to run 'git gc' in the work/ruby-builds directory before measuring disk usage,
    # since this will give git a chance to do de-duping and re-compression.
    #
    # Invoke this script with a since argument - the path to the local clone of
    # ruby.

    require 'fileutils'
    require 'rugged'

    class RubyBuilder
    attr_reader :repo_path

    def initialize(base_dir, repo_path)
    @base_dir = base_dir
    FileUtils.mkdir_p(base_dir)
    @repo_path = repo_path
    @src_archive_path = File.join(@base_dir, "ruby-dev.tar.gz")
    @builds_repo_path = File.join(@base_dir, "ruby-builds")
    @definitions_path = File.join(@base_dir, "definitions")

    @ruby_package_name = "ruby-dev"
    @build_definition_path = File.join(@definitions_path, @ruby_package_name)
    ensure_builds_repo
    end

    def ensure_builds_repo
    if !File.exist?(@builds_repo_path)
    FileUtils.mkdir_p(@builds_repo_path)
    system("cd #{@builds_repo_path} && git init")
    end
    end

    def archive_source(commit)
    prefix = "#{@ruby_package_name}/"
    puts "Archiving from #{@repo_path} to #{@src_archive_path}"
    cmd = "cd #{@repo_path} && git archive --format tar #{commit.oid} --prefix #{prefix} | gzip >#{@src_archive_path}"
    puts " #{cmd}"
    system(cmd)
    end

    def generate_build_definition
    definition = <<-END
    install_package "openssl-1.0.1e" "https://www.openssl.org/source/openssl-1.0.1e.tar.gz#66bf6f10f060d561929de96f9dfe5b8c" mac_openssl --if has_broken_mac_openssl
    install_package "#{@ruby_package_name}" "#{@src_archive_path}" ldflags_dirs autoconf standard verify_openssl
    END
    FileUtils.mkdir_p(@definitions_path)
    File.open(@build_definition_path, "w") do |f|
    f.write(definition)
    end
    end

    def with_env(env)
    old_values = {}
    env.keys.each { |key| old_values[key] = ENV[key] }
    env.each { |key, value| ENV[key] = value }
    yield
    old_values.each { |key, value| ENV[key] = value }
    end

    def invoke_ruby_build(commit)
    puts "Building Ruby at #{commit.oid} from #{@build_definition_path} into #{@builds_repo_path}"
    build_invocation = "ruby-build #{@build_definition_path} #{@builds_repo_path}"
    puts " #{build_invocation}"
    env = {
    'RUBY_BUILD_CACHE_PATH' => @base_dir,
    'CONFIGURE_OPTS' => '--disable-install-doc'
    }
    with_env(env) { system(build_invocation) }
    end

    def clean_builds_repo
    puts "Cleaning #{@builds_repo_path}"
    Dir[File.join(@builds_repo_path, "*")].each { |e| FileUtils.rmtree(e) }
    end

    def commit_to_builds_repo(src_commit)
    commit_msg = "Build of ruby from #{src_commit.oid}\n"
    commit_msg << "\n"
    commit_msg << "Original commit message:\n"
    commit_msg << "#{src_commit.message}"
    commit_msg_file = File.join(@base_dir, "commit.msg")
    File.open(commit_msg_file, "w") { |f| f.write(commit_msg) }

    puts "Committing to #{@builds_repo_path}"
    add_cmd = "cd #{@builds_repo_path} && git add --all ."
    system(add_cmd)
    commit_cmd = "cd #{@builds_repo_path} && git commit -F #{commit_msg_file}"
    system(commit_cmd)
    end

    def build_ruby_version(commit)
    archive_source(commit)
    generate_build_definition
    clean_builds_repo
    invoke_ruby_build(commit)
    commit_to_builds_repo(commit)
    end
    end

    local_ruby_repo_path = ARGV[0]
    work_dir = File.expand_path('./work')
    builder = RubyBuilder.new(work_dir, local_ruby_repo_path)

    repo = Rugged::Repository.new(local_ruby_repo_path)
    head_commit = Rugged::Branch.lookup(repo, "trunk").tip
    head_sha = head_commit.oid
    time_period = (60 * 60 * 24 * 365) # 1 year
    earliest_timestamp = head_commit.time.to_f - time_period

    walker = Rugged::Walker.new(repo)
    walker.sorting(Rugged::SORT_TOPO)
    walker.push(head_sha)
    walker.each do |c|
    break if c.time.to_f < earliest_timestamp
    builder.build_ruby_version(c)
    end