Skip to content

Instantly share code, notes, and snippets.

@jerinphilip

jerinphilip/Makefile

Last active March 24, 2021 18:58
Show Gist options
  • Save jerinphilip/f1caac37ac1eced45e6e9d94380bc4d0 to your computer and use it in GitHub Desktop.
Save jerinphilip/f1caac37ac1eced45e6e9d94380bc4d0 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. Jerin Philip revised this gist Mar 24, 2021. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -53,8 +53,8 @@ wasm: dirs bergamot
    emcmake $(CMAKE) -L \
    -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    -DCOMPILE_WASM=on \
    $(BERGAMOT)
    cd $(WASM_BUILD) && make -f $(WASM_BUILD)/Makefile -j$(THREADS)
    $(BERGAMOT) && \
    cd $(WASM_BUILD) && emmake make -f $(WASM_BUILD)/Makefile -j$(THREADS)

    native: dirs bergamot
    cd $(NATIVE_BUILD) && \
  2. Jerin Philip revised this gist Mar 22, 2021. 1 changed file with 4 additions and 7 deletions.
    11 changes: 4 additions & 7 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -47,23 +47,20 @@ models: dirs

    first-setup: emsdk dirs models

    wasm: bergamot dirs
    wasm: dirs bergamot
    $(EMSDK)/emsdk activate latest &> /tmp/error.log || cat /tmp/error.log
    source $(EMSDK)/emsdk_env.sh && cd $(WASM_BUILD) && \
    emcmake $(CMAKE) -L \
    -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    -DCOMPILE_WASM=on \
    -DPACKAGE_DIR=$(MODELS) \
    $(BERGAMOT)
    cd $(WASM_BUILD) && make -f $(WASM_BUILD)/Makefile -j$(THREADS)
    cd $(WASM_BUILD) && make -f $(WASM_BUILD)/Makefile -j$(THREADS)

    native: bergamot dirs
    native: dirs bergamot
    cd $(NATIVE_BUILD) && \
    $(CMAKE) \
    -L \
    -DCOMPILE_CUDA=off -DCOMPILE_DECODER_ONLY=on -DCOMPILE_LIBRARY_ONLY=off \
    -DUSE_MKL=on -DCOMPILE_THREAD_VARIANT=on -DCOMPILE_MAIN_EIGEN=off \
    -DONNXJS_SGEMM=off -DUSE_WASM_COMPATIBLE_BLAS=off \
    -DCOMPILE_CUDA=off -DUSE_WASM_COMPATIBLE_SOURCES=off\
    -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    $(BERGAMOT)

  3. Jerin Philip revised this gist Feb 17, 2021. 1 changed file with 6 additions and 3 deletions.
    9 changes: 6 additions & 3 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -59,11 +59,14 @@ wasm: bergamot dirs

    native: bergamot dirs
    cd $(NATIVE_BUILD) && \
    $(CMAKE) -L -DCOMPILE_CUDA=off -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    -DCOMPILE_DECODER_ONLY=on -DCOMPILE_LIBRARY_ONLY=off -DUSE_MKL=on \
    -DCOMPILE_THREAD_VARIANT=on -DCOMPILE_MAIN_EIGEN=off \
    $(CMAKE) \
    -L \
    -DCOMPILE_CUDA=off -DCOMPILE_DECODER_ONLY=on -DCOMPILE_LIBRARY_ONLY=off \
    -DUSE_MKL=on -DCOMPILE_THREAD_VARIANT=on -DCOMPILE_MAIN_EIGEN=off \
    -DONNXJS_SGEMM=off -DUSE_WASM_COMPATIBLE_BLAS=off \
    -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    $(BERGAMOT)

    cd $(NATIVE_BUILD) && make -f $(NATIVE_BUILD)/Makefile -j$(THREADS)

    clean:
  4. Jerin Philip revised this gist Feb 17, 2021. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -60,7 +60,7 @@ wasm: bergamot dirs
    native: bergamot dirs
    cd $(NATIVE_BUILD) && \
    $(CMAKE) -L -DCOMPILE_CUDA=off -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    -DCOMPILE_DECODER_ONLY=off -DCOMPILE_LIBRARY_ONLY=off -DUSE_MKL=on \
    -DCOMPILE_DECODER_ONLY=on -DCOMPILE_LIBRARY_ONLY=off -DUSE_MKL=on \
    -DCOMPILE_THREAD_VARIANT=on -DCOMPILE_MAIN_EIGEN=off \
    -DONNXJS_SGEMM=off -DUSE_WASM_COMPATIBLE_BLAS=off \
    $(BERGAMOT)
  5. Jerin Philip revised this gist Feb 17, 2021. 1 changed file with 3 additions and 1 deletion.
    4 changes: 3 additions & 1 deletion Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -61,7 +61,9 @@ native: bergamot dirs
    cd $(NATIVE_BUILD) && \
    $(CMAKE) -L -DCOMPILE_CUDA=off -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    -DCOMPILE_DECODER_ONLY=off -DCOMPILE_LIBRARY_ONLY=off -DUSE_MKL=on \
    -DCOMPILE_THREAD_VARIANT=on -S $(BERGAMOT)
    -DCOMPILE_THREAD_VARIANT=on -DCOMPILE_MAIN_EIGEN=off \
    -DONNXJS_SGEMM=off -DUSE_WASM_COMPATIBLE_BLAS=off \
    $(BERGAMOT)
    cd $(NATIVE_BUILD) && make -f $(NATIVE_BUILD)/Makefile -j$(THREADS)

    clean:
  6. Jerin Philip revised this gist Feb 17, 2021. 1 changed file with 4 additions and 3 deletions.
    7 changes: 4 additions & 3 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -58,9 +58,10 @@ wasm: bergamot dirs
    cd $(WASM_BUILD) && make -f $(WASM_BUILD)/Makefile -j$(THREADS)

    native: bergamot dirs
    cd $(NATIVE_BUILD) && $(CMAKE) -L -DCOMPILE_CUDA=off -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    -DCOMPILE_DECODER_ONLY=off -DCOMPILE_LIBRARY_ONLY=off -DUSE_MKL=on \
    -DCOMPILE_THREAD_VARIANT=on -S $(BERGAMOT)
    cd $(NATIVE_BUILD) && \
    $(CMAKE) -L -DCOMPILE_CUDA=off -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    -DCOMPILE_DECODER_ONLY=off -DCOMPILE_LIBRARY_ONLY=off -DUSE_MKL=on \
    -DCOMPILE_THREAD_VARIANT=on -S $(BERGAMOT)
    cd $(NATIVE_BUILD) && make -f $(NATIVE_BUILD)/Makefile -j$(THREADS)

    clean:
  7. Jerin Philip revised this gist Feb 17, 2021. 1 changed file with 8 additions and 5 deletions.
    13 changes: 8 additions & 5 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -5,16 +5,17 @@

    THREADS=40

    SHELL := /bin/bash
    SHELL := /bin/bash

    ROOT := /mnt/Storage/jphilip/bergamot
    ROOT := /mnt/Storage/jphilip/bergamot
    BERGAMOT := $(ROOT)/bergamot-translator
    EMSDK := $(ROOT)/emsdk
    MODELS := $(ROOT)/models

    BUILD := $(ROOT)/build
    BUILD := $(ROOT)/build
    NATIVE_BUILD := $(BUILD)/native
    WASM_BUILD := $(BUILD)/wasm
    MODELS_GZ := $(BUILD)/models.gz.d

    # Requires CMake > 3.12 or something. Locally compiled and added.
    # 3.20.0-rc1 is used while testing this script
    @@ -28,7 +29,7 @@ BRANCH ?= jp/absorb-batch-translator
    .PHONY: emsdk bergamot models dirs

    dirs:
    mkdir -p $(ROOT) $(BUILD) $(NATIVE_BUILD) $(WASM_BUILD)
    mkdir -p $(ROOT) $(BUILD) $(NATIVE_BUILD) $(WASM_BUILD) $(MODELS_GZ)

    emsdk:
    git -C $(EMSDK) pull || git clone https://github.com/emscripten-core/emsdk.git $(EMSDK)
    @@ -39,8 +40,10 @@ bergamot:
    git -C $(BERGAMOT) checkout $(BRANCH)
    git -C $(BERGAMOT) submodule update --init --recursive

    models:
    models: dirs
    git -C $(MODELS) pull || git clone https://github.com/mozilla-applied-ml/bergamot-models $(MODELS)
    cp -r $(MODELS)/* $(MODELS_GZ)/
    gunzip $(MODELS_GZ)/*/*

    first-setup: emsdk dirs models

  8. Jerin Philip revised this gist Feb 17, 2021. 1 changed file with 9 additions and 2 deletions.
    11 changes: 9 additions & 2 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -44,7 +44,7 @@ models:

    first-setup: emsdk dirs models

    wasm: bergamot
    wasm: bergamot dirs
    $(EMSDK)/emsdk activate latest &> /tmp/error.log || cat /tmp/error.log
    source $(EMSDK)/emsdk_env.sh && cd $(WASM_BUILD) && \
    emcmake $(CMAKE) -L \
    @@ -54,10 +54,17 @@ wasm: bergamot
    $(BERGAMOT)
    cd $(WASM_BUILD) && make -f $(WASM_BUILD)/Makefile -j$(THREADS)

    native: bergamot
    native: bergamot dirs
    cd $(NATIVE_BUILD) && $(CMAKE) -L -DCOMPILE_CUDA=off -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    -DCOMPILE_DECODER_ONLY=off -DCOMPILE_LIBRARY_ONLY=off -DUSE_MKL=on \
    -DCOMPILE_THREAD_VARIANT=on -S $(BERGAMOT)
    cd $(NATIVE_BUILD) && make -f $(NATIVE_BUILD)/Makefile -j$(THREADS)

    clean:
    rm $(BUILD) -rv

    clean-native:
    rm $(NATIVE_BUILD) -rv

    clean-wasm:
    rm $(WASM_BUILD) -rv
  9. Jerin Philip revised this gist Feb 17, 2021. 1 changed file with 15 additions and 7 deletions.
    22 changes: 15 additions & 7 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -19,6 +19,7 @@ WASM_BUILD := $(BUILD)/wasm
    # Requires CMake > 3.12 or something. Locally compiled and added.
    # 3.20.0-rc1 is used while testing this script
    CMAKE := /home/jphilip/.local/bin/cmake
    BUILD_TYPE ?= Release

    # Parameterize builds by branches
    BRANCH ?= jp/absorb-batch-translator
    @@ -35,19 +36,26 @@ emsdk:

    bergamot:
    git -C $(BERGAMOT) pull || git clone https://github.com/browsermt/bergamot-translator $(BERGAMOT)
    git -C $(BERGAMOT) checkout $(BRANCH)
    git -C $(BERGAMOT) submodule update --init --recursive

    models:
    git -C $(MODELS) pull || git clone https://github.com/mozilla-applied-ml/bergamot-models $(MODELS)

    wasm: emsdk bergamot models dirs
    $(EMSDK)/emsdk activate latest
    git -C $(BERGAMOT) checkout $(BRANCH)
    source $(EMSDK)/emsdk_env.sh && cd $(WASM_BUILD) && emcmake $(CMAKE) DCOMPILE_WASM=on -DPACKAGE_DIR=$(MODELS) $(BERGAMOT)
    first-setup: emsdk dirs models

    wasm: bergamot
    $(EMSDK)/emsdk activate latest &> /tmp/error.log || cat /tmp/error.log
    source $(EMSDK)/emsdk_env.sh && cd $(WASM_BUILD) && \
    emcmake $(CMAKE) -L \
    -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    -DCOMPILE_WASM=on \
    -DPACKAGE_DIR=$(MODELS) \
    $(BERGAMOT)
    cd $(WASM_BUILD) && make -f $(WASM_BUILD)/Makefile -j$(THREADS)

    native: dirs bergamot
    git -C $(BERGAMOT) checkout $(BRANCH)
    cd $(NATIVE_BUILD) && $(CMAKE) -DCOMPILE_CUDA=off -DCMAKE_BUILD_TYPE=Release \
    native: bergamot
    cd $(NATIVE_BUILD) && $(CMAKE) -L -DCOMPILE_CUDA=off -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
    -DCOMPILE_DECODER_ONLY=off -DCOMPILE_LIBRARY_ONLY=off -DUSE_MKL=on \
    -DCOMPILE_THREAD_VARIANT=on -S $(BERGAMOT)
    cd $(NATIVE_BUILD) && make -f $(NATIVE_BUILD)/Makefile -j$(THREADS)
  10. Jerin Philip revised this gist Feb 17, 2021. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -21,7 +21,7 @@ WASM_BUILD := $(BUILD)/wasm
    CMAKE := /home/jphilip/.local/bin/cmake

    # Parameterize builds by branches
    BRANCH := jp/absorb-batch-translator
    BRANCH ?= jp/absorb-batch-translator


    .PHONY: emsdk bergamot models dirs
  11. Jerin Philip revised this gist Feb 17, 2021. 1 changed file with 15 additions and 4 deletions.
    19 changes: 15 additions & 4 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,6 @@
    # This Makefile is being prepared to cache installation setups on
    # [email protected] and ensure that WASM pipeline builds and route
    # through regression tests.


    THREADS=40
    @@ -13,6 +16,14 @@ BUILD := $(ROOT)/build
    NATIVE_BUILD := $(BUILD)/native
    WASM_BUILD := $(BUILD)/wasm

    # Requires CMake > 3.12 or something. Locally compiled and added.
    # 3.20.0-rc1 is used while testing this script
    CMAKE := /home/jphilip/.local/bin/cmake

    # Parameterize builds by branches
    BRANCH := jp/absorb-batch-translator


    .PHONY: emsdk bergamot models dirs

    dirs:
    @@ -30,13 +41,13 @@ models:

    wasm: emsdk bergamot models dirs
    $(EMSDK)/emsdk activate latest
    git -C $(BERGAMOT) checkout wasm-integration
    source $(EMSDK)/emsdk_env.sh && cd $(WASM_BUILD) && emcmake cmake DCOMPILE_WASM=on -DPACKAGE_DIR=$(MODELS) $(BERGAMOT)
    git -C $(BERGAMOT) checkout $(BRANCH)
    source $(EMSDK)/emsdk_env.sh && cd $(WASM_BUILD) && emcmake $(CMAKE) DCOMPILE_WASM=on -DPACKAGE_DIR=$(MODELS) $(BERGAMOT)
    cd $(WASM_BUILD) && make -f $(WASM_BUILD)/Makefile -j$(THREADS)

    native: dirs bergamot
    git -C $(BERGAMOT) checkout jp/absorb-batch-translator
    cd $(NATIVE_BUILD) && cmake -DCOMPILE_CUDA=off -DCMAKE_BUILD_TYPE=Release \
    git -C $(BERGAMOT) checkout $(BRANCH)
    cd $(NATIVE_BUILD) && $(CMAKE) -DCOMPILE_CUDA=off -DCMAKE_BUILD_TYPE=Release \
    -DCOMPILE_DECODER_ONLY=off -DCOMPILE_LIBRARY_ONLY=off -DUSE_MKL=on \
    -DCOMPILE_THREAD_VARIANT=on -S $(BERGAMOT)
    cd $(NATIVE_BUILD) && make -f $(NATIVE_BUILD)/Makefile -j$(THREADS)
  12. jerinphilip created this gist Feb 17, 2021.
    44 changes: 44 additions & 0 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,44 @@


    THREADS=40

    SHELL := /bin/bash

    ROOT := /mnt/Storage/jphilip/bergamot
    BERGAMOT := $(ROOT)/bergamot-translator
    EMSDK := $(ROOT)/emsdk
    MODELS := $(ROOT)/models

    BUILD := $(ROOT)/build
    NATIVE_BUILD := $(BUILD)/native
    WASM_BUILD := $(BUILD)/wasm

    .PHONY: emsdk bergamot models dirs

    dirs:
    mkdir -p $(ROOT) $(BUILD) $(NATIVE_BUILD) $(WASM_BUILD)

    emsdk:
    git -C $(EMSDK) pull || git clone https://github.com/emscripten-core/emsdk.git $(EMSDK)
    $(EMSDK)/emsdk install latest

    bergamot:
    git -C $(BERGAMOT) pull || git clone https://github.com/browsermt/bergamot-translator $(BERGAMOT)

    models:
    git -C $(MODELS) pull || git clone https://github.com/mozilla-applied-ml/bergamot-models $(MODELS)

    wasm: emsdk bergamot models dirs
    $(EMSDK)/emsdk activate latest
    git -C $(BERGAMOT) checkout wasm-integration
    source $(EMSDK)/emsdk_env.sh && cd $(WASM_BUILD) && emcmake cmake DCOMPILE_WASM=on -DPACKAGE_DIR=$(MODELS) $(BERGAMOT)
    cd $(WASM_BUILD) && make -f $(WASM_BUILD)/Makefile -j$(THREADS)

    native: dirs bergamot
    git -C $(BERGAMOT) checkout jp/absorb-batch-translator
    cd $(NATIVE_BUILD) && cmake -DCOMPILE_CUDA=off -DCMAKE_BUILD_TYPE=Release \
    -DCOMPILE_DECODER_ONLY=off -DCOMPILE_LIBRARY_ONLY=off -DUSE_MKL=on \
    -DCOMPILE_THREAD_VARIANT=on -S $(BERGAMOT)
    cd $(NATIVE_BUILD) && make -f $(NATIVE_BUILD)/Makefile -j$(THREADS)