Skip to content

Instantly share code, notes, and snippets.

@dinigo

dinigo/difference

Last active February 3, 2017 09:41
Show Gist options
  • Save dinigo/066eb7fc053d9fd48be1e9a1b932121c to your computer and use it in GitHub Desktop.
Save dinigo/066eb7fc053d9fd48be1e9a1b932121c to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. dinigo revised this gist Feb 3, 2017. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -2,7 +2,7 @@
    # setopt shwordsplit

    # Script para calcular las altas y bajas en cada mes entre dos archivos
    # para vodafone


    #===================================================================#
    # #
  2. dinigo revised this gist Feb 3, 2017. 2 changed files with 6 additions and 3 deletions.
    7 changes: 5 additions & 2 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -39,11 +39,11 @@ ROW="" # columna en que se encuentra el id a encruptar
    SECONDS=$(date +%s) # fecha en segundos en que se comenzó a ejecutar el script
    PARAMS=$# # numero de parametros con que se invoco el script
    GIST_URL="https://api.github.com/gists/066eb7fc053d9fd48be1e9a1b932121c"
    DEPENDENCIES="r gdate jq wget curl" # dependencias de la aplicacion
    DEPENDENCIES="coreutils r jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="11" # version de la app, tambien se usa para update
    APP_VERSION="12" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    @@ -107,6 +107,9 @@ check_dependencies(){
    brew tap homebrew/science
    brew install Caskroom/cask/xquartz
    ;;
    coreutils)
    brew tap homebrew/dupes
    ;;
    esac
    brew install $dep --with-default-names
    event "install" "finished" "dependency" "$dep"
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    11
    12
  3. dinigo revised this gist Feb 3, 2017. 2 changed files with 7 additions and 5 deletions.
    10 changes: 6 additions & 4 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -39,18 +39,20 @@ ROW="" # columna en que se encuentra el id a encruptar
    SECONDS=$(date +%s) # fecha en segundos en que se comenzó a ejecutar el script
    PARAMS=$# # numero de parametros con que se invoco el script
    GIST_URL="https://api.github.com/gists/066eb7fc053d9fd48be1e9a1b932121c"
    DEPENDENCIES="r jq wget curl" # dependencias de la aplicacion
    DEPENDENCIES="r gdate jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="10" # version de la app, tambien se usa para update
    APP_VERSION="11" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual

    # Alias
    date () { gdate "$@"; }

    # Si estamos en mac usamos gdate en lugar de date
    if [[ $(uname) -eq 'Darwin' ]]; then
    date () { gdate "$@"; }
    fi


    #===================================================================#
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    10
    11
  4. dinigo revised this gist Feb 3, 2017. 2 changed files with 4 additions and 6 deletions.
    8 changes: 3 additions & 5 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -43,16 +43,14 @@ DEPENDENCIES="r jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="9" # version de la app, tambien se usa para update
    APP_VERSION="10" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual

    # Alias
    # Si estamos en mac usamos gdate en lugar de date
    if [[ $(uname) -eq 'Darwin' ]]; then
    date () { gdate "$@"; }
    fi
    date () { gdate "$@"; }



    #===================================================================#
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    9
    10
  5. dinigo revised this gist Feb 2, 2017. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -85,7 +85,7 @@ check_dependencies(){
    if [[ ! -f difference ]]; then
    print dark_gray "Descargando script para cruzar altas y bajas"
    local gist=$(curl -s $GIST_URL)
    local dif_url=$(echo $gist | jq -r '.files.log_template.raw_url')
    local dif_url=$(echo $gist | jq -r '.files.difference.raw_url')
    curl -s -o difference $dif_url
    chmod +x difference
    fi
  6. dinigo revised this gist Feb 2, 2017. 1 changed file with 13 additions and 9 deletions.
    22 changes: 13 additions & 9 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -76,6 +76,19 @@ cleanup(){
    # Comprueba que se cumplen las dependencias del programa y las instala
    check_dependencies(){
    print dark_gray "Comprobando dependencias"
    if [[ ! -f $LOG_FILE ]]; then
    print dark_gray "Descargando plantilla de log"
    local gist=$(curl -s $GIST_URL)
    local log_url=$(echo $gist | jq -r '.files.log_template.raw_url')
    curl -s -o $LOG_FILE $log_url
    fi
    if [[ ! -f difference ]]; then
    print dark_gray "Descargando script para cruzar altas y bajas"
    local gist=$(curl -s $GIST_URL)
    local dif_url=$(echo $gist | jq -r '.files.log_template.raw_url')
    curl -s -o difference $dif_url
    chmod +x difference
    fi
    if ! command -v brew >/dev/null 2>&1; then
    print grey "---------------------------"
    print white " Instalando gestor de paquetes"
    @@ -138,9 +151,6 @@ check_upgrade(){
    curl -s -o dmp_upload $script_url
    mv $LOG_FILE back_$LOG_FILE
    curl -s -o $LOG_FILE $log_url
    [[ -f difference ]] && rm difference
    curl -s -o difference $difference_url
    chmod +x difference
    event "update" "finished" "script" "$latest"
    log green "Ejecutando la nueva version"
    chmod +x dmp_upload
    @@ -173,12 +183,6 @@ log(){
    # Crea los tags necesarios para el log
    open_log(){
    event "function" "run" "open_log"
    if [[ ! -f $LOG_FILE ]]; then
    print dark_gray "Descargando plantilla de log"
    local gist=$(curl -s $GIST_URL)
    local log_url=$(echo $gist | jq -r '.files.log_template.raw_url')
    curl -s -o $LOG_FILE $log_url
    fi
    # agrega el indice
    local title="$(date +"%y-%m-%d %H:%M")"
    local id=entry-$SECONDS
  7. dinigo revised this gist Feb 2, 2017. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -43,7 +43,7 @@ DEPENDENCIES="r jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="8" # version de la app, tambien se usa para update
    APP_VERSION="9" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    @@ -140,7 +140,7 @@ check_upgrade(){
    curl -s -o $LOG_FILE $log_url
    [[ -f difference ]] && rm difference
    curl -s -o difference $difference_url
    chmod +x difference $difference_url
    chmod +x difference
    event "update" "finished" "script" "$latest"
    log green "Ejecutando la nueva version"
    chmod +x dmp_upload
  8. dinigo revised this gist Feb 2, 2017. 2 changed files with 2 additions and 2 deletions.
    2 changes: 1 addition & 1 deletion dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -139,7 +139,7 @@ check_upgrade(){
    mv $LOG_FILE back_$LOG_FILE
    curl -s -o $LOG_FILE $log_url
    [[ -f difference ]] && rm difference
    curl -s -o difference
    curl -s -o difference $difference_url
    chmod +x difference $difference_url
    event "update" "finished" "script" "$latest"
    log green "Ejecutando la nueva version"
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    8
    9
  9. dinigo revised this gist Feb 2, 2017. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -140,7 +140,7 @@ check_upgrade(){
    curl -s -o $LOG_FILE $log_url
    [[ -f difference ]] && rm difference
    curl -s -o difference
    chmod +x difference
    chmod +x difference $difference_url
    event "update" "finished" "script" "$latest"
    log green "Ejecutando la nueva version"
    chmod +x dmp_upload
  10. dinigo revised this gist Feb 2, 2017. 3 changed files with 58 additions and 18 deletions.
    25 changes: 25 additions & 0 deletions difference
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,25 @@
    #!/usr/bin/env Rscript
    args = commandArgs(trailingOnly=TRUE)

    base_file <- paste0(getwd(),'/',args[1])
    delta_file <- paste0(getwd(),'/',args[2])
    altas_file <- paste0(getwd(),'/',args[3])
    bajas_file <- paste0(getwd(),'/',args[4])

    cat('cargando base... \n');
    base <- read.csv(base_file,sep=',',header=FALSE)
    base <- t(base)

    cat('cargando delta... \n');
    delta <- read.csv(delta_file,sep=',',header=FALSE)
    delta <- t(delta)

    cat('calculando bajas... \n')
    bajas <- base[!base%in%delta]
    cat('calculando altas.. \n')
    altas <- delta[!delta%in%base]

    cat('escribiendo bajas... \n');
    write.csv(altas, file = altas_file,row.names=FALSE,quote=FALSE)
    cat('escribiendo altas... \n');
    write.csv(bajas, file = bajas_file,row.names=FALSE,quote=FALSE)
    49 changes: 32 additions & 17 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -39,15 +39,21 @@ ROW="" # columna en que se encuentra el id a encruptar
    SECONDS=$(date +%s) # fecha en segundos en que se comenzó a ejecutar el script
    PARAMS=$# # numero de parametros con que se invoco el script
    GIST_URL="https://api.github.com/gists/066eb7fc053d9fd48be1e9a1b932121c"
    DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion
    DEPENDENCIES="r jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="7" # version de la app, tambien se usa para update
    APP_VERSION="8" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual

    # Alias
    # Si estamos en mac usamos gdate en lugar de date
    if [[ $(uname) -eq 'Darwin' ]]; then
    date () { gdate "$@"; }
    fi


    #===================================================================#
    # #
    @@ -83,6 +89,12 @@ check_dependencies(){
    if ! brew list $dep >/dev/null 2>&1; then
    print white " Instalando dependencia: $dep"
    event "install" "start" "dependency" "$dep"
    case $dep in
    r)
    brew tap homebrew/science
    brew install Caskroom/cask/xquartz
    ;;
    esac
    brew install $dep --with-default-names
    event "install" "finished" "dependency" "$dep"
    else
    @@ -95,13 +107,20 @@ check_dependencies(){
    spinner(){
    local pid=$1
    local delay=0.1
    local start_time=$(date +%s)
    local end_time=''
    local seconds=''
    local elapsed=''
    while [ $(ps -eo pid | grep $pid) ]; do
    end_time=$(date +%s)
    seconds=$(($end_time - $start_time))
    elapsed=$(date -u -d @${seconds} +"%T")
    for i in \| / - \\; do
    printf ' procesando [%c]\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b' $i
    printf ' procesando %s [%c]\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b' $elapsed $i
    sleep $delay
    done
    done
    printf '\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b'
    printf '\b\b\b\b \n'
    }

    # Comprueba si hay aactualizaciones, las descarga y actualiza
    @@ -112,12 +131,16 @@ check_upgrade(){
    local latest=$(echo $gist | jq -r '.files.version.content')
    local script_url=$(echo $gist | jq -r '.files.dmp_upload.raw_url')
    local log_url=$(echo $gist | jq -r '.files.log_template.raw_url')
    local difference_url=$(echo $gist | jq -r '.files.difference.raw_url')
    if [[ $latest -gt $APP_VERSION ]]; then
    event "update" "start" "script" "$latest"
    log green "Actualizando de la version $APP_VERSION a la $latest"
    curl -s -o dmp_upload $script_url
    mv $LOG_FILE back_$LOG_FILE
    curl -s -o $LOG_FILE $log_url
    [[ -f difference ]] && rm difference
    curl -s -o difference
    chmod +x difference
    event "update" "finished" "script" "$latest"
    log green "Ejecutando la nueva version"
    chmod +x dmp_upload
    @@ -564,11 +587,11 @@ else
    pageview "prepare-multi-file"
    log grey "---------------------------"
    #1 Extrae las bajas
    log white " Extrae las bajas"
    (grep -F -v -f $DELTA $BASE > bajas.csv) &
    log white " Extrae altas y bajas"
    (./difference $BASE $DELTA altas.csv bajas.csv) &
    spinner $!
    TEMP_FILES+=" bajas.csv"
    log cyan " Bajas: $(cat bajas.csv | wc -l)"
    TEMP_FILES+=" bajas.csv altas.csv"
    log cyan " Altas: $(cat altas.csv | wc -l), Bajas: $(cat bajas.csv | wc -l)"
    #2 Encripta las bajas con los id de trait que eliminar
    pageview "extract-unscribers"
    log grey "---------------------------"
    @@ -580,14 +603,6 @@ else
    log green "$(head -n 4 bajas-encrypted.csv)"
    mv bajas-encrypted.csv bajas.csv
    $UPLOAD && ask
    log grey "---------------------------"
    #3 Extrae las altas
    pageview "extract-subscribers"
    log white " Extrae las altas"
    (grep -F -v -f $BASE $DELTA > altas.csv) &
    spinner $!
    TEMP_FILES+=" altas.csv"
    log cyan " Altas: $(cat altas.csv | wc -l)"
    #4 Encripta las altas
    pageview "process-subscribers"
    log grey "---------------------------"
    @@ -643,4 +658,4 @@ if $UPLOAD; then
    user $FTP_USER $FTP_PASS
    put $OUTPUT
    bye" | ftp -n > ftp_$$.log
    fi
    fi
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    7
    8
  11. dinigo revised this gist Jan 31, 2017. 2 changed files with 70 additions and 35 deletions.
    103 changes: 69 additions & 34 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -19,6 +19,7 @@ COMPRESS=false # flag para comprimir
    UPLOAD=false # flag para subir los archivos tras transformarlos
    HASH=false # flag para calcular el hash de los ids
    BAD_PARAMS=false # flag para mostrar la ayuda despues de parsear los params
    DEBUG=false # flag para mostrar información de debug

    # Archivos
    BASE="" # archivo base sobre el que se aplican incrementos
    @@ -42,7 +43,7 @@ DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="6" # version de la app, tambien se usa para update
    APP_VERSION="7" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    @@ -56,28 +57,42 @@ APP_PATH="" # variable acumulador path para el estado actual
    # #
    #===================================================================#


    # Elimina todos los archivos en la lista de temporales
    cleanup(){
    if [[ $CLEAN ]]; then
    for file in $TEMP_FILES; do
    [ -f $file ] && rm $file
    done
    fi
    }

    # Comprueba que se cumplen las dependencias del programa y las instala
    function check_dependencies(){
    event "function" "run" "check_dependencies"
    log dark_gray "Comprobando dependencias"
    check_dependencies(){
    print dark_gray "Comprobando dependencias"
    if ! command -v brew >/dev/null 2>&1; then
    log grey "---------------------------"
    log white " Instalando gestor de paquetes"
    print grey "---------------------------"
    print white " Instalando gestor de paquetes"
    event "install" "start" "dependency" "brew"
    /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
    event "install" "finished" "dependency" "brew"
    else
    $DEBUG && print grey " Brew instalado"
    fi
    for dep in $DEPENDENCIES;do
    if ! brew list $dep >/dev/null 2>&1; then
    log white " Instalando dependencia: $dep"
    print white " Instalando dependencia: $dep"
    event "install" "start" "dependency" "$dep"
    brew install $dep --with-default-names
    event "install" "finished" "dependency" "$dep"
    else
    $DEBUG && print grey " Dependencia satisfecha: $dep"
    fi
    done
    }

    function spinner(){
    # Muestra una animación y el tiempo que lleva ejecutándose el último proceso
    spinner(){
    local pid=$1
    local delay=0.1
    while [ $(ps -eo pid | grep $pid) ]; do
    @@ -90,7 +105,7 @@ function spinner(){
    }

    # Comprueba si hay aactualizaciones, las descarga y actualiza
    function check_upgrade(){
    check_upgrade(){
    event "function" "run" "check_upgrade"
    log dark_gray "Comprobando actualizaciones"
    local gist=$(curl -s $GIST_URL)
    @@ -112,7 +127,7 @@ function check_upgrade(){
    }

    # Procesa el mensaje a loguear
    function log(){
    log(){
    local code=$1
    local message=$2
    case $code in
    @@ -125,15 +140,22 @@ function log(){
    open)
    event "function" "open" "log"
    open_log;;
    * )
    *)
    append_log $code "$message"
    $VERBOSE && print $code "$message"
    ;;
    esac
    }

    # Crea los tags necesarios para el log
    function open_log(){
    open_log(){
    event "function" "run" "open_log"
    if [[ ! -f $LOG_FILE ]]; then
    print dark_gray "Descargando plantilla de log"
    local gist=$(curl -s $GIST_URL)
    local log_url=$(echo $gist | jq -r '.files.log_template.raw_url')
    curl -s -o $LOG_FILE $log_url
    fi
    # agrega el indice
    local title="$(date +"%y-%m-%d %H:%M")"
    local id=entry-$SECONDS
    @@ -146,7 +168,7 @@ function open_log(){
    }

    # Agrega un log
    function append_log(){
    append_log(){
    local color=$1
    local message=$2
    local text="<span style=\"color:$color\">$message</span>"
    @@ -155,7 +177,7 @@ function append_log(){
    }

    # Agrega texto al html justo al final de cierta etiqueta
    function append_html(){
    append_html(){
    local content=$1
    local tags=$2
    local closer=""
    @@ -167,7 +189,7 @@ function append_html(){
    }

    # Imprime por terminal el mensaje desseado a color
    function print(){
    print(){
    # extrae los parametros
    local color=$1
    local message="$2"
    @@ -194,7 +216,8 @@ function print(){
    echo -e "\033[${COLOR}${message}\033[0m"
    }

    function ask(){
    # Pregunta si los datos son correctos para continuar el programa
    ask(){
    event "function" "run" "ask" "$message"
    while true; do
    echo "Son correctos estos datos? (s|n):"
    @@ -208,7 +231,7 @@ function ask(){
    }

    # Imprime la ayuda del programa
    function help(){
    help(){
    event "function" "run" "help"
    echo "./prepare-dmp.sh [argumentos]"
    echo " -a, --datasource: id del datasource a usar en el archivo de salida"
    @@ -226,6 +249,7 @@ function help(){
    echo " -w, --row: fila en que se encuentra el msisdn"
    echo " -x, --execute: cadena a enviar para hacer match"
    echo " -m, --md5: calcula el hash md5 de los IDs"
    echo " -u, --debug: modo debug"
    echo ""
    echo " Ejemplo de uso:"
    echo " $./prepare-upload.sh \\"
    @@ -240,7 +264,7 @@ function help(){
    }

    # Almacena y obtiene un id a modo de cookie
    function get_uuid() {
    get_uuid() {
    local client_id_file=${1:-$HOME/.uuid}
    local client_id=""

    @@ -255,13 +279,10 @@ function get_uuid() {


    # Envia analitica a google
    function analytics() {

    analytics() {
    DOMAIN="www.google-analytics.com"
    ANALYTICS_PATH="collect"
    # ANALYTICS_PATH="debug/collect"
    $DEBUG && ANALYTICS_PATH="debug/collect" || ANALYTICS_PATH="collect"
    PAYLOAD="v=1&cid=$(get_uuid)"

    while [[ $# > 0 ]] ; do
    case "$1" in
    -t|--t|--type)
    @@ -317,12 +338,15 @@ function analytics() {
    done

    URL="https://$DOMAIN/$ANALYTICS_PATH?$PAYLOAD"
    curl -s -X POST --data "$PAYLOAD" $URL > /dev/null&
    # curl -X POST --data "$PAYLOAD" $URL
    if $DEBUG; then
    curl -X POST --data "$PAYLOAD" $URL
    else
    curl -s -X POST --data "$PAYLOAD" $URL > /dev/null&
    fi
    }

    # Abstrae la llamada a la vista de pagina
    function pageview(){
    pageview(){
    local step=$1
    APP_PATH+="/$step"
    analytics \
    @@ -337,7 +361,7 @@ function pageview(){
    }

    # Abstre la llamada a un evento
    function event(){
    event(){
    local command="--t event --tid $TRACKING_ID --an $APP_NAME --av $APP_VERSION --aid $APP_ID"
    [[ -n $1 ]] && command+=" --ec $1"
    [[ -n $2 ]] && command+=" --ea $2"
    @@ -348,15 +372,24 @@ function event(){

    #===================================================================#
    # #
    # Parse a los parametros de entrada para construir la config #
    # Bloque de inicialización y configuracion #
    # #
    #===================================================================#

    pageview "start"
    trap cleanup SIGHUP SIGINT SIGTERM
    check_dependencies
    log open
    pageview "start"
    check_upgrade
    check_dependencies
    VERBOSE=false


    #===================================================================#
    # #
    # Parse a los parametros de entrada para construir la config #
    # #
    #===================================================================#

    # parsea los parametros que se pasan al script
    while [[ $# > 0 ]] ; do
    case "$1" in
    @@ -445,6 +478,10 @@ while [[ $# > 0 ]] ; do
    COMPRESS=true # si se va a subir es necesario comprimirlo
    log dark_gray "Sube los archivos al ftp"
    ;;
    -u|--debug)
    DEBUG=true
    log dark_gray "Modo debug activado"
    ;;
    esac
    shift
    done
    @@ -591,9 +628,7 @@ if $CLEAN; then
    log grey "---------------------------"
    log white " Elimina archivos temporales"
    echo $TEMP_FILES
    for file in $TEMP_FILES; do
    [ -f $file ] && rm $file
    done
    cleanup
    fi

    if $UPLOAD; then
    @@ -608,4 +643,4 @@ if $UPLOAD; then
    user $FTP_USER $FTP_PASS
    put $OUTPUT
    bye" | ftp -n > ftp_$$.log
    fi
    fi
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    6
    7
  12. dinigo revised this gist Jan 19, 2017. 1 changed file with 3 additions and 3 deletions.
    6 changes: 3 additions & 3 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -42,7 +42,7 @@ DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="4" # version de la app, tambien se usa para update
    APP_VERSION="6" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    @@ -98,15 +98,15 @@ function check_upgrade(){
    local script_url=$(echo $gist | jq -r '.files.dmp_upload.raw_url')
    local log_url=$(echo $gist | jq -r '.files.log_template.raw_url')
    if [[ $latest -gt $APP_VERSION ]]; then
    mv $LOG_FILE back_$LOG_FILE
    event "update" "start" "script" "$latest"
    log green "Actualizando de la version $APP_VERSION a la $latest"
    curl -s -o dmp_upload $script_url
    mv $LOG_FILE back_$LOG_FILE
    curl -s -o $LOG_FILE $log_url
    event "update" "finished" "script" "$latest"
    log green "Ejecutando la nueva version"
    chmod +x dmp_upload
    ./$filename $ARGS
    ./dmp_upload $ARGS
    exit 0
    fi
    }
  13. dinigo revised this gist Jan 19, 2017. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -98,14 +98,14 @@ function check_upgrade(){
    local script_url=$(echo $gist | jq -r '.files.dmp_upload.raw_url')
    local log_url=$(echo $gist | jq -r '.files.log_template.raw_url')
    if [[ $latest -gt $APP_VERSION ]]; then
    mv "$LOG_FILE back_$LOG_FILE"
    mv $LOG_FILE back_$LOG_FILE
    event "update" "start" "script" "$latest"
    log green "Actualizando de la version $APP_VERSION a la $latest"
    curl -s -o dmp_upload $script_url
    curl -s -o $LOG_FILE $log_url
    event "update" "finished" "script" "$latest"
    log green "Ejecutando la nueva version"
    chmod +x $filename
    chmod +x dmp_upload
    ./$filename $ARGS
    exit 0
    fi
  14. dinigo revised this gist Jan 19, 2017. 2 changed files with 3 additions and 3 deletions.
    4 changes: 2 additions & 2 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -24,7 +24,7 @@ BAD_PARAMS=false # flag para mostrar la ayuda despues de parsear los params
    BASE="" # archivo base sobre el que se aplican incrementos
    DELTA="" # archivo incremental
    OUTPUT="" # nombre del archivo de salida (opcioneal)
    LOG_FILE=log # archivo de log por defecto
    LOG_FILE=log.html # archivo de log por defecto

    # Config
    SCRIPT_NAME="$0" # nombre del script
    @@ -103,7 +103,7 @@ function check_upgrade(){
    log green "Actualizando de la version $APP_VERSION a la $latest"
    curl -s -o dmp_upload $script_url
    curl -s -o $LOG_FILE $log_url
    vent "update" "finished" "script" "$latest"
    event "update" "finished" "script" "$latest"
    log green "Ejecutando la nueva version"
    chmod +x $filename
    ./$filename $ARGS
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    5
    6
  15. dinigo revised this gist Jan 19, 2017. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    4
    5
  16. dinigo revised this gist Jan 19, 2017. 2 changed files with 2 additions and 2 deletions.
    2 changes: 1 addition & 1 deletion dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -42,7 +42,7 @@ DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="5" # version de la app, tambien se usa para update
    APP_VERSION="4" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    5
    4
  17. dinigo revised this gist Jan 19, 2017. 3 changed files with 115 additions and 13 deletions.
    26 changes: 14 additions & 12 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -24,7 +24,7 @@ BAD_PARAMS=false # flag para mostrar la ayuda despues de parsear los params
    BASE="" # archivo base sobre el que se aplican incrementos
    DELTA="" # archivo incremental
    OUTPUT="" # nombre del archivo de salida (opcioneal)
    LOG_FILE=log.html # archivo de log por defecto
    LOG_FILE=log # archivo de log por defecto

    # Config
    SCRIPT_NAME="$0" # nombre del script
    @@ -42,7 +42,7 @@ DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="4" # version de la app, tambien se usa para update
    APP_VERSION="5" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    @@ -95,13 +95,15 @@ function check_upgrade(){
    log dark_gray "Comprobando actualizaciones"
    local gist=$(curl -s $GIST_URL)
    local latest=$(echo $gist | jq -r '.files.version.content')
    local download_url=$(echo $gist | jq -r '.files.dmp_upload.raw_url')
    local filename=$(echo $gist | jq -r '.files.dmp_upload.filename')
    local script_url=$(echo $gist | jq -r '.files.dmp_upload.raw_url')
    local log_url=$(echo $gist | jq -r '.files.log_template.raw_url')
    if [[ $latest -gt $APP_VERSION ]]; then
    mv "$LOG_FILE back_$LOG_FILE"
    event "update" "start" "script" "$latest"
    log green "Actualizando de la version $APP_VERSION a la $latest"
    curl -s -o $filename $download_url
    event "update" "finished" "script" "$latest"
    curl -s -o dmp_upload $script_url
    curl -s -o $LOG_FILE $log_url
    vent "update" "finished" "script" "$latest"
    log green "Ejecutando la nueva version"
    chmod +x $filename
    ./$filename $ARGS
    @@ -137,8 +139,8 @@ function open_log(){
    local id=entry-$SECONDS
    local entry="<li><a href=\"#$id\">$title</a></li>"
    local entry_tag="ul"
    local pre="</pre><pre id=\"$id\"><span>$title</span>"
    local pre_tag="article pre"
    local pre="<pre id=\"$id\"><span>$title</span></pre>"
    local pre_tag="body div div"
    append_html "$entry" "$entry_tag"
    append_html "$pre" "$pre_tag"
    }
    @@ -147,8 +149,8 @@ function open_log(){
    function append_log(){
    local color=$1
    local message=$2
    local text="</span><span style=\"color:$color\">$message"
    local selector="article pre span"
    local text="<span style=\"color:$color\">$message</span>"
    local selector="div div pre"
    append_html "$text" "$selector"
    }

    @@ -158,10 +160,10 @@ function append_html(){
    local tags=$2
    local closer=""
    for tag in $tags;do
    closer="</$tag>$closer"
    closer="</$tag>\s*$closer"
    done
    content="${content//$'\n'/\\n}" #sustituye nuevas lineas por el caracter
    sed -i -e "s|$closer|\n$content$closer|g" $LOG_FILE
    sed -r -i -e "s|(\s*)($closer)|\n\1$content\1\2|g" $LOG_FILE
    }

    # Imprime por terminal el mensaje desseado a color
    100 changes: 100 additions & 0 deletions log_template
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,100 @@
    <!DOCTYPE html>
    <html>
    <head>
    <style>
    * {
    font-family: "Arial", Helvetica, sans-serif;
    }
    body {
    margin:0;
    }
    pre {
    padding:4em;
    margin:1em;
    padding:1em;
    background:#444;
    border-radius: 3px;
    display: inline-block;
    margin: 1rem;
    position: relative;
    transition: box-shadow 0.3s cubic-bezier(.25,.8,.25,1);
    box-shadow: 0 1px 3px rgba(0,0,0,0.12), 0 1px 2px rgba(0,0,0,0.24);
    }
    pre:hover {
    box-shadow: 0 14px 28px rgba(0,0,0,0.25), 0 10px 10px rgba(0,0,0,0.22);
    }
    pre, pre>span {
    font-family: "Lucida Console", Monaco, monospace;
    }
    pre>span:first-of-type {
    font-weight: bolder;
    color: grey;
    }
    a {
    color: #00ACD6;
    }
    footer {background: #00ACD6;color:white;}
    ul {
    list-style-type: none;
    padding: 0;
    }
    li {
    margin: 20px;
    }
    ul a {
    text-decoration: none;
    margin: 10px;
    padding: 10px;
    }
    ul a:hover {
    background-color: #444;
    color: white;
    }
    #wrapper {
    width: 100%;
    background-color: #fff;
    }
    #top-nav {
    position: fixed;
    left: 0;
    right: 0;
    top: 0;
    height: 5em;
    width: 100%;
    text-align: center;
    background-color: black;
    color:white;
    }
    #side-nav {
    position: fixed;
    width: 13em
    height:100vh;
    left: 0;
    top: 5em;
    overflow: auto;
    background-color: white;
    }
    #content-wrapper {
    overflow: auto;
    position: fixed;
    left: 13em;
    right: 0;
    top: 5em;
    height:100vh;
    background: #e2e1e0;
    }
    </style>
    </head>
    <body>
    <div id="wrapper">
    <div id="top-nav">
    <h1>Logs del proceso de archivos de CRM</h1>
    </div>
    <div id="side-nav">
    <ul>
    <li><a href="#inicio">Inicio</a></li>
    </ul>
    </div>
    <div id="content-wrapper">
    <pre id="inicio"><span>Inicio de los logs</span></pre></div></div></body>
    </html>
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    4
    5
  18. dinigo revised this gist Jan 19, 2017. 2 changed files with 30 additions and 10 deletions.
    38 changes: 29 additions & 9 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -42,7 +42,7 @@ DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="3" # version de la app, tambien se usa para update
    APP_VERSION="4" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    @@ -59,6 +59,7 @@ APP_PATH="" # variable acumulador path para el estado actual
    # Comprueba que se cumplen las dependencias del programa y las instala
    function check_dependencies(){
    event "function" "run" "check_dependencies"
    log dark_gray "Comprobando dependencias"
    if ! command -v brew >/dev/null 2>&1; then
    log grey "---------------------------"
    log white " Instalando gestor de paquetes"
    @@ -76,10 +77,22 @@ function check_dependencies(){
    done
    }

    function spinner(){
    local pid=$1
    local delay=0.1
    while [ $(ps -eo pid | grep $pid) ]; do
    for i in \| / - \\; do
    printf ' procesando [%c]\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b' $i
    sleep $delay
    done
    done
    printf '\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b'
    }

    # Comprueba si hay aactualizaciones, las descarga y actualiza
    function check_upgrade(){
    event "function" "run" "check_upgrade"
    log dark_gray "Comprobando actualizaciones"
    local gist=$(curl -s $GIST_URL)
    local latest=$(echo $gist | jq -r '.files.version.content')
    local download_url=$(echo $gist | jq -r '.files.dmp_upload.raw_url')
    @@ -485,8 +498,10 @@ fi
    # Ejecuta el programa perse: #
    # Si solo hay archivo base opera con el #
    # Si hay archivo delta calcula las altas y bajas #
    # Ejecuta el programa perse: #
    # Ejecuta el programa perse: #
    # Procesa altas y bajas #
    # Pasa a minuscula o no #
    # Comprime el archivo y limpia #
    # Sube el archivo al DMP #
    # #
    #===================================================================#

    @@ -495,13 +510,14 @@ if [ -z $DELTA ]; then
    pageview "prepare-single-file"
    log grey "---------------------------"
    log white " Encripta el archivo de entrada con los traits"
    java -jar msisdn.jar \
    (java -jar msisdn.jar \
    $COMMAND \
    --input $BASE \
    --output $OUTPUT \
    $REMOVE_TRAITS \
    $TRAITS \
    $EXECUTE
    $EXECUTE) &
    spinner $!
    log white " Ejemplo de linea encriptada:"
    log green "$(head -n 4 $OUTPUT)"
    $UPLOAD && ask
    @@ -510,14 +526,16 @@ else
    log grey "---------------------------"
    #1 Extrae las bajas
    log white " Extrae las bajas"
    grep -F -v -f $DELTA $BASE | pv -tpreb > bajas.csv
    (grep -F -v -f $DELTA $BASE > bajas.csv) &
    spinner $!
    TEMP_FILES+=" bajas.csv"
    log cyan " Bajas: $(cat bajas.csv | wc -l)"
    #2 Encripta las bajas con los id de trait que eliminar
    pageview "extract-unscribers"
    log grey "---------------------------"
    log white " Procesa las bajas con los traits a desabilitar"
    java -jar msisdn.jar $COMMAND -o bajas-encrypted.csv -i bajas.csv $REMOVE_TRAITS
    (java -jar msisdn.jar $COMMAND -o bajas-encrypted.csv -i bajas.csv $REMOVE_TRAITS) &
    spinner $!
    pageview "process-unscribers"
    log white " Ejemplo de bajas encriptadas:"
    log green "$(head -n 4 bajas-encrypted.csv)"
    @@ -527,14 +545,16 @@ else
    #3 Extrae las altas
    pageview "extract-subscribers"
    log white " Extrae las altas"
    grep -F -v -f $BASE $DELTA | pv > altas.csv
    (grep -F -v -f $BASE $DELTA > altas.csv) &
    spinner $!
    TEMP_FILES+=" altas.csv"
    log cyan " Altas: $(cat altas.csv | wc -l)"
    #4 Encripta las altas
    pageview "process-subscribers"
    log grey "---------------------------"
    log white " Procesa las altas"
    java -jar msisdn.jar $COMMAND -i altas.csv -o altas-encrypted.csv $TRAITS $EXECUTE $EXECUTE
    (java -jar msisdn.jar $COMMAND -i altas.csv -o altas-encrypted.csv $TRAITS $EXECUTE $EXECUTE) &
    spinner $!
    log white " Ejemplo de altas encriptadas:"
    log green "$(head -n 4 altas-encrypted.csv)"
    $UPLOAD && ask
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    3
    4
  19. dinigo revised this gist Jan 19, 2017. 2 changed files with 5 additions and 5 deletions.
    8 changes: 4 additions & 4 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -42,7 +42,7 @@ DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="2" # version de la app, tambien se usa para update
    APP_VERSION="3" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    @@ -303,7 +303,7 @@ function analytics() {

    URL="https://$DOMAIN/$ANALYTICS_PATH?$PAYLOAD"
    curl -s -X POST --data "$PAYLOAD" $URL > /dev/null&
    # curl -v -X POST --data "$PAYLOAD" $URL
    # curl -X POST --data "$PAYLOAD" $URL
    }

    # Abstrae la llamada a la vista de pagina
    @@ -312,7 +312,7 @@ function pageview(){
    APP_PATH+="/$step"
    analytics \
    --type pageview \
    --trace-id $TRACKING_ID \
    --tracking-id $TRACKING_ID \
    --document-hostname "dmp-upload" \
    --document-path "$APP_PATH" \
    --document-title "$step" \
    @@ -510,7 +510,7 @@ else
    log grey "---------------------------"
    #1 Extrae las bajas
    log white " Extrae las bajas"
    grep -F -v -f $DELTA $BASE | pv > bajas.csv
    grep -F -v -f $DELTA $BASE | pv -tpreb > bajas.csv
    TEMP_FILES+=" bajas.csv"
    log cyan " Bajas: $(cat bajas.csv | wc -l)"
    #2 Encripta las bajas con los id de trait que eliminar
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    2
    3
  20. dinigo revised this gist Jan 19, 2017. 2 changed files with 5 additions and 4 deletions.
    7 changes: 4 additions & 3 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -42,7 +42,7 @@ DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="1" # version de la app, tambien se usa para update
    APP_VERSION="2" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    @@ -427,6 +427,7 @@ while [[ $# > 0 ]] ; do
    -u|--upload)
    UPLOAD=true
    VERBOSE=true
    COMPRESS=true # si se va a subir es necesario comprimirlo
    log dark_gray "Sube los archivos al ftp"
    ;;
    esac
    @@ -509,7 +510,7 @@ else
    log grey "---------------------------"
    #1 Extrae las bajas
    log white " Extrae las bajas"
    grep -F -v -f $DELTA $BASE > bajas.csv
    grep -F -v -f $DELTA $BASE | pv > bajas.csv
    TEMP_FILES+=" bajas.csv"
    log cyan " Bajas: $(cat bajas.csv | wc -l)"
    #2 Encripta las bajas con los id de trait que eliminar
    @@ -526,7 +527,7 @@ else
    #3 Extrae las altas
    pageview "extract-subscribers"
    log white " Extrae las altas"
    grep -F -v -f $BASE $DELTA > altas.csv
    grep -F -v -f $BASE $DELTA | pv > altas.csv
    TEMP_FILES+=" altas.csv"
    log cyan " Altas: $(cat altas.csv | wc -l)"
    #4 Encripta las altas
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    1
    2
  21. dinigo revised this gist Jan 11, 2017. 2 changed files with 1 addition and 2 deletions.
    1 change: 0 additions & 1 deletion dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -1,6 +1,5 @@
    #!/bin/bash
    # setopt shwordsplit
    echo ultima version

    # Script para calcular las altas y bajas en cada mes entre dos archivos
    # para vodafone
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    4
    1
  22. dinigo revised this gist Jan 11, 2017. 2 changed files with 2 additions and 1 deletion.
    1 change: 1 addition & 0 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -1,5 +1,6 @@
    #!/bin/bash
    # setopt shwordsplit
    echo ultima version

    # Script para calcular las altas y bajas en cada mes entre dos archivos
    # para vodafone
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    1
    4
  23. dinigo revised this gist Jan 11, 2017. 2 changed files with 2 additions and 2 deletions.
    2 changes: 1 addition & 1 deletion dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -42,7 +42,7 @@ DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="4" # version de la app, tambien se usa para update
    APP_VERSION="1" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    4
    1
  24. dinigo revised this gist Jan 11, 2017. No changes.
  25. dinigo revised this gist Jan 11, 2017. 2 changed files with 2 additions and 2 deletions.
    2 changes: 1 addition & 1 deletion dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -42,7 +42,7 @@ DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="3" # version de la app, tambien se usa para update
    APP_VERSION="4" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    3
    4
  26. dinigo revised this gist Jan 11, 2017. 2 changed files with 2 additions and 2 deletions.
    2 changes: 1 addition & 1 deletion dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -42,7 +42,7 @@ DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="1" # version de la app, tambien se usa para update
    APP_VERSION="3" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    2 changes: 1 addition & 1 deletion version
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    2
    3
  27. dinigo revised this gist Jan 11, 2017. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -1,5 +1,5 @@
    #!/bin/bash
    setopt shwordsplit
    # setopt shwordsplit

    # Script para calcular las altas y bajas en cada mes entre dos archivos
    # para vodafone
  28. dinigo revised this gist Jan 11, 2017. 1 changed file with 40 additions and 27 deletions.
    67 changes: 40 additions & 27 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -1,5 +1,5 @@
    #!/bin/bash
    # setopt shwordsplit
    setopt shwordsplit

    # Script para calcular las altas y bajas en cada mes entre dos archivos
    # para vodafone
    @@ -38,11 +38,11 @@ ROW="" # columna en que se encuentra el id a encruptar
    SECONDS=$(date +%s) # fecha en segundos en que se comenzó a ejecutar el script
    PARAMS=$# # numero de parametros con que se invoco el script
    GIST_URL="https://api.github.com/gists/066eb7fc053d9fd48be1e9a1b932121c"
    DEPENDENCIES="jq wget curl" # dependencias de la aplicacion
    DEPENDENCIES="gnu-sed grep jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="2" # version de la app, tambien se usa para update
    APP_VERSION="1" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual
    @@ -56,6 +56,27 @@ APP_PATH="" # variable acumulador path para el estado actual
    # #
    #===================================================================#

    # Comprueba que se cumplen las dependencias del programa y las instala
    function check_dependencies(){
    event "function" "run" "check_dependencies"
    if ! command -v brew >/dev/null 2>&1; then
    log grey "---------------------------"
    log white " Instalando gestor de paquetes"
    event "install" "start" "dependency" "brew"
    /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
    event "install" "finished" "dependency" "brew"
    fi
    for dep in $DEPENDENCIES;do
    if ! brew list $dep >/dev/null 2>&1; then
    log white " Instalando dependencia: $dep"
    event "install" "start" "dependency" "$dep"
    brew install $dep --with-default-names
    event "install" "finished" "dependency" "$dep"
    fi
    done
    }


    # Comprueba si hay aactualizaciones, las descarga y actualiza
    function check_upgrade(){
    event "function" "run" "check_upgrade"
    @@ -64,8 +85,10 @@ function check_upgrade(){
    local download_url=$(echo $gist | jq -r '.files.dmp_upload.raw_url')
    local filename=$(echo $gist | jq -r '.files.dmp_upload.filename')
    if [[ $latest -gt $APP_VERSION ]]; then
    event "update" "start" "script" "$latest"
    log green "Actualizando de la version $APP_VERSION a la $latest"
    curl -s -o $filename $download_url
    event "update" "finished" "script" "$latest"
    log green "Ejecutando la nueva version"
    chmod +x $filename
    ./$filename $ARGS
    @@ -286,7 +309,7 @@ function analytics() {
    # Abstrae la llamada a la vista de pagina
    function pageview(){
    local step=$1
    APP_PATH+=":$step"
    APP_PATH+="/$step"
    analytics \
    --type pageview \
    --trace-id $TRACKING_ID \
    @@ -314,9 +337,10 @@ function event(){
    # #
    #===================================================================#

    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "start" --dt "start"
    pageview "start"
    log open
    check_upgrade
    check_dependencies
    VERBOSE=false
    # parsea los parametros que se pasan al script
    while [[ $# > 0 ]] ; do
    @@ -467,8 +491,7 @@ fi

    # Si no hay archivo delta trata el archivo base solamente
    if [ -z $DELTA ]; then
    APP_PATH+=":prepare-single-file"
    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "$APP_PATH" --dt "Single File"
    pageview "prepare-single-file"
    log grey "---------------------------"
    log white " Encripta el archivo de entrada con los traits"
    java -jar msisdn.jar \
    @@ -482,37 +505,32 @@ if [ -z $DELTA ]; then
    log green "$(head -n 4 $OUTPUT)"
    $UPLOAD && ask
    else
    APP_PATH+=":prepare-multi-file"
    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "$APP_PATH" --dt "Multi File"
    pageview "prepare-multi-file"
    log grey "---------------------------"
    #1 Extrae las bajas
    log white " Extrae las bajas"
    grep -F -v -f $DELTA $BASE > bajas.csv
    TEMP_FILES+=" bajas.csv"
    log cyan " Bajas: $(cat bajas.csv | wc -l)"
    #2 Encripta las bajas con los id de trait que eliminar
    APP_PATH+=":extract-unscribers"
    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "$APP_PATH" --dt "Extract Unscribers"
    pageview "extract-unscribers"
    log grey "---------------------------"
    log white " Procesa las bajas con los traits a desabilitar"
    java -jar msisdn.jar $COMMAND -o bajas-encrypted.csv -i bajas.csv $REMOVE_TRAITS
    APP_PATH+=":process-unscribers"
    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "$APP_PATH" --dt "Process Unscribers"
    pageview "process-unscribers"
    log white " Ejemplo de bajas encriptadas:"
    log green "$(head -n 4 bajas-encrypted.csv)"
    mv bajas-encrypted.csv bajas.csv
    $UPLOAD && ask
    log grey "---------------------------"
    #3 Extrae las altas
    APP_PATH+=":extract-subscribers"
    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "$APP_PATH" --dt "Extract Subscribers"
    pageview "extract-subscribers"
    log white " Extrae las altas"
    grep -F -v -f $BASE $DELTA > altas.csv
    TEMP_FILES+=" altas.csv"
    log cyan " Altas: $(cat altas.csv | wc -l)"
    #4 Encripta las altas
    APP_PATH+=":process-subscribers"
    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "$APP_PATH" --dt "Process Subscribers"
    pageview "process-subscribers"
    log grey "---------------------------"
    log white " Procesa las altas"
    java -jar msisdn.jar $COMMAND -i altas.csv -o altas-encrypted.csv $TRAITS $EXECUTE $EXECUTE
    @@ -522,35 +540,31 @@ else
    mv altas-encrypted.csv altas.csv
    log grey "---------------------------"
    #6 Concatena los archivos de altas y bajas listos
    APP_PATH+=":join-files"
    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "$APP_PATH" --dt "Join Files"
    pageview "join-files"
    log white " Concatena las altas y bajas"
    cat bajas.csv altas.csv > $OUTPUT
    log cyan " Archivo de salida: $OUTPUT"
    log cyan " Total: $(cat $OUTPUT | wc -l)"
    fi

    if $LOWER; then
    APP_PATH+=":lowercase"
    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "$APP_PATH" --dt "To Lowercase"
    pageview "lowercase"
    log grey "---------------------------"
    log white " Pasa a minuscula"
    tr A-Z a-z < $OUTPUT > lower_$OUTPUT
    mv lower_$OUTPUT $OUTPUT
    fi

    if $COMPRESS; then
    APP_PATH+=":compress"
    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "$APP_PATH" --dt "Compress"
    pageview "compress"
    zip -9 $OUTPUT.zip $OUTPUT
    TEMP_FILES+=" $OUTPUT"
    OUTPUT=$OUTPUT.zip
    fi

    # Limpia los archivos temporales si esta activado
    if $CLEAN; then
    APP_PATH+=":clean"
    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "$APP_PATH" --dt "Clean"
    pageview "clean"
    log grey "---------------------------"
    log white " Elimina archivos temporales"
    echo $TEMP_FILES
    @@ -560,8 +574,7 @@ if $CLEAN; then
    fi

    if $UPLOAD; then
    APP_PATH+=":upload"
    analytics -t pageview --tid $TRACKING_ID --dh "dmp-upload" --dp "$APP_PATH" --dt "Upload"
    pageview "upload"
    log grey "---------------------------"
    log white " Subida de archivos"
    log white " Comprueba que las muestras de archivo tienen sentido"
  29. Daniel Iñigo revised this gist Jan 10, 2017. 1 changed file with 0 additions and 2 deletions.
    2 changes: 0 additions & 2 deletions dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -1,8 +1,6 @@
    #!/bin/bash
    # setopt shwordsplit

    echo SEGUNDA VERSION

    # Script para calcular las altas y bajas en cada mes entre dos archivos
    # para vodafone

  30. Daniel Iñigo revised this gist Jan 10, 2017. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion dmp_upload
    View file
    Original file line number Diff line number Diff line change
    @@ -44,7 +44,7 @@ DEPENDENCIES="jq wget curl" # dependencias de la aplicacion

    # Analitica
    TRACKING_ID="UA-69106054-6" # propiedad de google analytics
    APP_VERSION="1" # version de la app, tambien se usa para update
    APP_VERSION="2" # version de la app, tambien se usa para update
    APP_NAME="DMP Upload" # nombre de la app
    APP_ID="dmp-upload:shell" # nombre descriptivo de la app
    APP_PATH="" # variable acumulador path para el estado actual