Skip to content

Instantly share code, notes, and snippets.

@erichiggins

erichiggins/zipimportify.py

Last active December 10, 2016 10:02
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save erichiggins/fd125afff7edf82bd7f7 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save erichiggins/fd125afff7edf82bd7f7 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. erichiggins revised this gist Nov 16, 2015. No changes.
  2. erichiggins created this gist Nov 13, 2015.
    102 changes: 102 additions & 0 deletions zipimportify.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,102 @@
    #!/usr/bin/env python
    """
    Convert a directory of pip-installed libraries into zip-imports for use on GAE.
    Usage:
    pip install -U <library_name>
    pip freeze > requirements_dev.txt
    pip install -U --egg --target <path_to_zips> -r requirements_dev.txt
    python zipimportify.py <path_to_zips>
    cd <path_to_zips>
    ls | grep -v '\.zip' | xargs rm -rf
    """
    import argparse
    import json
    import os
    import subprocess
    import sys


    PIP_INFO_SUFFIX = '-info'
    TOP_LEVEL = 'top_level.txt'
    METADATA = 'metadata.json'

    parser = argparse.ArgumentParser(description='Zip up and version pip-installed libraries.')
    parser.add_argument('path', type=str)


    def top_level(path):
    with open(os.path.join(path, TOP_LEVEL), 'r') as fp:
    line = fp.readline().strip()
    # Note(eric): Packages like PyYaml have top_level lines like "__yaml" that should be skipped.
    while line.startswith('_'):
    line = fp.readline().strip()
    return line


    def version(path):
    with open(os.path.join(path, METADATA), 'r') as fp:
    metadata = json.load(fp)
    return metadata['version']


    def zip_dir(dir_to_zip, filename, cwd=None):
    return subprocess.Popen(['zip', '-qr0', filename, dir_to_zip], cwd=cwd, universal_newlines=True)

    def zip_files(files_to_zip, filename, cwd=None):
    return subprocess.Popen(['zip', '-qr0', filename, '.', '-i', files_to_zip], cwd=cwd, universal_newlines=True)


    def main(args):
    # Everything in the directory.
    all_files = set(os.listdir(args.path))
    # Everything that ends with "-info".
    info_names = set([x for x in all_files if x.endswith(PIP_INFO_SUFFIX)])
    print len(info_names), '-info files:'
    print info_names
    print

    # Create paths by prepending the directory.
    info_paths = set([os.path.join(args.path, x) for x in info_names])
    print len(info_paths), 'info paths:'
    print info_paths
    print

    # Every package version that matches a subdirectory name.
    versions = {top_level(x): version(x) for x in info_paths}
    print len(versions.keys()), 'versions:'
    print versions
    print

    packages = set([x for x in versions.iterkeys() if os.path.isdir(os.path.join(args.path, x))])
    print len(packages), 'packages:'
    print packages
    print

    non_dir_packages = set(versions.iterkeys()) - packages
    print len(non_dir_packages), 'non_dir_packages:'
    print non_dir_packages
    print

    # Create a zip for each package.
    print 'zip paths:'
    for path in packages:
    zip_path = path + '-' + versions[path] + '.zip'
    print path, '>', zip_path
    zip_dir(path, zip_path, cwd=args.path)
    print

    # Create a zip for each non-directory package.
    print 'zip paths:'
    for path in non_dir_packages:
    zip_path = path + '-' + versions[path] + '.zip'
    path = path + '.py*'
    print path, '>', zip_path
    zip_files(path, zip_path, cwd=args.path)
    print


    if __name__ == '__main__':
    args = parser.parse_args()
    main(args)