Skip to content

Instantly share code, notes, and snippets.

@nestukh

nestukh/manga_checker.sh

Last active March 4, 2023 18:33
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save nestukh/5397b836c8e5f34f6feb4ec4efe6b86a to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save nestukh/5397b836c8e5f34f6feb4ec4efe6b86a to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. nestukh revised this gist Jun 15, 2019. 1 changed file with 12 additions and 10 deletions.
    22 changes: 12 additions & 10 deletions manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -240,19 +240,21 @@ if [[ ! ( -d "$HOME/bin/comics_dowloander_python_en" ) ]]; then
    pip install --upgrade --no-cache-dir beautifulsoup4 regex
    deactivate
    fi

    if [[ ! ( -d "${tfolderprov}/manga_tmp" ) ]]; then
    mkdir -p "${tfolderprov}/manga_tmp" # comics-downloader folder https://github.com/Girbons/comics-downloader
    wget -O "${tfolderprov}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g" | head -n1)"
    chmod +x "${tfolderprov}/manga_tmp/comics-downloader"
    if [[ ! ( -d "${temp_folder}/manga_tmp" ) ]]; then
    mkdir -p "${temp_folder}/manga_tmp" # comics-downloader folder https://github.com/Girbons/comics-downloader
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader/releases/ | grep -P "Girbons.comics-downloader.releases.download.v.*.comics-downloader" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/arm\"/g;s/armv6l/arm\"/g;s/x86_64/\/comics-downloader\"/g")" | head -n1 | sed "s/^.*href=\"/https:\/\/github.com/g;s/\".*$//g")"
    chmod +x "${temp_folder}/manga_tmp/comics-downloader"
    else
    cd "${tfolderprov}/manga_tmp"
    current_version="$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g" | head -n1 | sed "s/^.*\/download\///g;s/\/.*$//g")"
    cd "${temp_folder}/manga_tmp"
    current_version="$(wget -q -O- https://github.com/Girbons/comics-downloader/releases/ | grep -P "Girbons.comics-downloader.releases.download.v.*.comics-downloader" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/arm\"/g;s/armv6l/arm\"/g;s/x86_64/\/comics-downloader\"/g")" | head -n1 | sed "s/^.*\/download\///g;s/\/.*$//g")"
    installed_version="$(./comics-downloader --version | sed "s/^.* //g" | sed "s/[ \t]$//g")"
    if [[ "$installed_version" == "" ]]; then
    echo "$installed_version" >> "${temp_folder}/comics-downloader_error.txt"
    fi
    if [[ "$current_version" != "$installed_version" ]]; then
    rm -f "${tfolderprov}/manga_tmp/comics-downloader"
    wget -O "${tfolderprov}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g" | head -n1)"
    chmod +x "${tfolderprov}/manga_tmp/comics-downloader"
    rm -f "${temp_folder}/manga_tmp/comics-downloader"
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader/releases/ | grep -P "Girbons.comics-downloader.releases.download.v.*.comics-downloader" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/arm\"/g;s/armv6l/arm\"/g;s/x86_64/\/comics-downloader\"/g")" | head -n1 | sed "s/^.*href=\"/https:\/\/github.com/g;s/\".*$//g")"
    chmod +x "${temp_folder}/manga_tmp/comics-downloader"
    fi
    cd -
    fi
  2. nestukh revised this gist Jun 4, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -254,7 +254,7 @@ else
    wget -O "${tfolderprov}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g" | head -n1)"
    chmod +x "${tfolderprov}/manga_tmp/comics-downloader"
    fi
    cd
    cd -
    fi


  3. nestukh revised this gist Jun 4, 2019. 1 changed file with 14 additions and 4 deletions.
    18 changes: 14 additions & 4 deletions manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -241,10 +241,20 @@ if [[ ! ( -d "$HOME/bin/comics_dowloander_python_en" ) ]]; then
    deactivate
    fi

    if [[ ! ( -d "${temp_folder}/manga_tmp" ) ]]; then
    mkdir -p "${temp_folder}/manga_tmp" # comics-downloader folder
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g" | head -n1)"
    chmod +x "${temp_folder}/manga_tmp/comics-downloader"
    if [[ ! ( -d "${tfolderprov}/manga_tmp" ) ]]; then
    mkdir -p "${tfolderprov}/manga_tmp" # comics-downloader folder https://github.com/Girbons/comics-downloader
    wget -O "${tfolderprov}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g" | head -n1)"
    chmod +x "${tfolderprov}/manga_tmp/comics-downloader"
    else
    cd "${tfolderprov}/manga_tmp"
    current_version="$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g" | head -n1 | sed "s/^.*\/download\///g;s/\/.*$//g")"
    installed_version="$(./comics-downloader --version | sed "s/^.* //g" | sed "s/[ \t]$//g")"
    if [[ "$current_version" != "$installed_version" ]]; then
    rm -f "${tfolderprov}/manga_tmp/comics-downloader"
    wget -O "${tfolderprov}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g" | head -n1)"
    chmod +x "${tfolderprov}/manga_tmp/comics-downloader"
    fi
    cd
    fi


  4. nestukh revised this gist Jun 4, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -243,7 +243,7 @@ fi

    if [[ ! ( -d "${temp_folder}/manga_tmp" ) ]]; then
    mkdir -p "${temp_folder}/manga_tmp" # comics-downloader folder
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g")"
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g" | head -n1)"
    chmod +x "${temp_folder}/manga_tmp/comics-downloader"
    fi

  5. nestukh revised this gist Jun 4, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -243,7 +243,7 @@ fi

    if [[ ! ( -d "${temp_folder}/manga_tmp" ) ]]; then
    mkdir -p "${temp_folder}/manga_tmp" # comics-downloader folder
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm64\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g")"
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g")"
    chmod +x "${temp_folder}/manga_tmp/comics-downloader"
    fi

  6. nestukh revised this gist Jun 4, 2019. 1 changed file with 16 additions and 5 deletions.
    21 changes: 16 additions & 5 deletions manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -332,14 +332,25 @@ for manga in "${manganames[@]}"; do
    cd "${temp_folder}/manga_tmp"
    j="$(cat "/tmp/manga_incrj")"
    filex=""
    o=1
    fileok=true
    while [[ "$filex" == "" ]]; do # to prevent errors
    echo "[$j/$totalj/$q1/$k1]: downloading issue [$j] of [$totalj] on link $1 for $manga [$q1] (total number of mangas to download: [$k1])"
    ./comics-downloader -url="$1" -format=pdf
    if [[ "$o" -gt 10 ]]; then # in case of big errors
    # echo "$1" >> "${SHARED_SYNCTHING_FOLDER}/${manga}_lostchapter.txt"
    fileok=false
    break
    fi
    formatx=cbz # pdf, cbr, epub
    echo "[$j/$totalj/$q1/$k1]: downloading issue [$j] of [$totalj] on link $1 for $manga #[$q1] (total number of mangas to download: [$k1])"
    ./comics-downloader -url="$1" -format="$formatx"
    sleep 3
    filex="$(find $PWD -mindepth 1 -type f -name "*.pdf" -mmin -$((6*60)) -exec ls {} \; | head -n1)"
    filex="$(find $PWD -mindepth 1 -type f -name "*.$formatx" -mmin -$((6*60)) -exec ls {} \; | head -n1)"
    let "o=o+1";
    done
    # mv "$filex" "${SHARED_SYNCTHING_FOLDER}/$manga - $(basename "${filex%.*}").pdf" ######## this line move the pdf file to the shared synchting folder, or dropbox folder or else
    echo "$1" >> "$manga_lastpdfs"
    if [[ "$fileok" == "true" ]]; then
    # mv "$filex" "${SHARED_SYNCTHING_FOLDER}/$manga - $(basename "${filex%.*}").$formatx" ######## this line move the pdf file to the shared synchting folder, or dropbox folder or else
    echo "$1" >> "$manga_lastpdfs"
    fi
    let "j=j+1";
    echo "$j" > "/tmp/manga_incrj"
    }
  7. nestukh revised this gist Jun 3, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -322,7 +322,7 @@ for manga in "${manganames[@]}"; do
    mangaissueslist() {
    source $HOME/bin/comics_dowloander_python_env/bin/activate
    wget --tries=256 --timeout=256 --wait=$(( ( RANDOM % 10 ) + 1 )) --waitretry=$(( ( RANDOM % 10 ) + 1 )) --random-wait --no-dns-cache -O- -q "$mangapage" > /tmp/input.html
    python3 -c "import sys;import re;from bs4 import BeautifulSoup;pagex=open(\"/tmp/input.html\");pagey=pagex.read();soup=BeautifulSoup(pagey, 'html.parser');alla=soup.find_all(\"a\",text=True,href=True);[sys.stdout.write('https://mangarock.com'+line['href']+'\n') for line in alla if re.search(\"/manga/mrs-serie-.*/chapter/mrs-chapter\", line['href'])]"
    python3 -B -c "import sys;import re;from bs4 import BeautifulSoup;pagex=open(\"/tmp/input.html\");pagey=pagex.read();soup=BeautifulSoup(pagey, 'html.parser');alla=soup.find_all(\"a\",text=True,href=True);[sys.stdout.write('https://mangarock.com'+line['href']+'\n') for line in alla if re.search(\"/manga/mrs-serie-.*/chapter/mrs-chapter\", line['href'])]"
    rm -f /tmp/input.html
    deactivate
    }
  8. nestukh revised this gist Jun 3, 2019. 1 changed file with 1 addition and 0 deletions.
    1 change: 1 addition & 0 deletions manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -384,3 +384,4 @@ for manga in "${manganames[@]}"; do
    let "q1=q+1"
    done

    exit 0
  9. nestukh revised this gist Jun 3, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -241,7 +241,7 @@ if [[ ! ( -d "$HOME/bin/comics_dowloander_python_en" ) ]]; then
    deactivate
    fi

    if [[ !(-d "${temp_folder}/manga_tmp") ]]; then
    if [[ ! ( -d "${temp_folder}/manga_tmp" ) ]]; then
    mkdir -p "${temp_folder}/manga_tmp" # comics-downloader folder
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm64\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g")"
    chmod +x "${temp_folder}/manga_tmp/comics-downloader"
  10. nestukh revised this gist Jun 2, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -46,7 +46,7 @@

    # # # #
    # # # # pypcks="virtualenv python3-pip python3 python3-all-dev python3-dev libffi-dev libssl-dev librtmp-dev python3 python3-setuptools cython build-essential"
    # # # # allgoodpcks="inotify-tools ssh ntpdate unattended-upgrades"
    # # # # allgoodpcks="coreutils inotify-tools ssh ntpdate unattended-upgrades"
    # # # # sudo apt-get install -y $allgoodpcks $pypcks
    # # # #
    # # # # ### search last commands with up arrow key:
  11. nestukh revised this gist Jun 2, 2019. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -349,7 +349,7 @@ for manga in "${manganames[@]}"; do
    if [[ ! ( -f "$manga_lastpdfs" ) ]]; then
    touch "$manga_lastpdfs"
    rm -f "/tmp/manga_lastpdfs"
    mangaissueslist > "/tmp/manga_lastpdfs"
    mangaissueslist | sort -u | uniq > "/tmp/manga_lastpdfs"
    unset manga_link
    j=1
    echo "$j" > "/tmp/manga_incrj"
    @@ -362,7 +362,7 @@ for manga in "${manganames[@]}"; do
    else
    # check if new issue
    newlist=$(mangaissueslist)
    newlines=$(diff --old-line-format="" --new-line-format="%L" --unchanged-line-format="" <(cat "$manga_lastpdfs") <(echo -e "$newlist"))
    newlines=$(diff --old-line-format="" --new-line-format="%L" --unchanged-line-format="" <(cat "$manga_lastpdfs" | sort -u | uniq ) <(echo -e "$newlist" | sort -u | uniq ))
    if [[ "$newlines" != "" ]]; then
    echo "new issue for this manga!"
    unset manga_link
  12. nestukh revised this gist Jun 2, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -333,7 +333,7 @@ for manga in "${manganames[@]}"; do
    j="$(cat "/tmp/manga_incrj")"
    filex=""
    while [[ "$filex" == "" ]]; do # to prevent errors
    echo "[$j/$totalj/$q1/$k1]: dowloading issue [$j] of [$totalj] on link $1 for $manga [$q1] (total number of mangas to download: [$k1])"
    echo "[$j/$totalj/$q1/$k1]: downloading issue [$j] of [$totalj] on link $1 for $manga [$q1] (total number of mangas to download: [$k1])"
    ./comics-downloader -url="$1" -format=pdf
    sleep 3
    filex="$(find $PWD -mindepth 1 -type f -name "*.pdf" -mmin -$((6*60)) -exec ls {} \; | head -n1)"
  13. nestukh revised this gist Jun 2, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -243,7 +243,7 @@ fi

    if [[ !(-d "${temp_folder}/manga_tmp") ]]; then
    mkdir -p "${temp_folder}/manga_tmp" # comics-downloader folder
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspian-arm64\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g")"
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspbian-arm64\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g")"
    chmod +x "${temp_folder}/manga_tmp/comics-downloader"
    fi

  14. nestukh revised this gist Jun 1, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -243,7 +243,7 @@ fi

    if [[ !(-d "${temp_folder}/manga_tmp") ]]; then
    mkdir -p "${temp_folder}/manga_tmp" # comics-downloader folder
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspian-arm64\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g")"
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -v "\-gui" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspian-arm64\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/>Linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g")"
    chmod +x "${temp_folder}/manga_tmp/comics-downloader"
    fi

  15. nestukh revised this gist Jun 1, 2019. 1 changed file with 1 addition and 10 deletions.
    11 changes: 1 addition & 10 deletions manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -327,15 +327,6 @@ for manga in "${manganames[@]}"; do
    deactivate
    }
    export -f mangaissueslist

    mangapagedownload() {
    source $HOME/bin/comics_dowloander_python_env/bin/activate
    wget --tries=256 --timeout=256 --wait=$(( ( RANDOM % 10 ) + 1 )) --waitretry=$(( ( RANDOM % 10 ) + 1 )) --random-wait --no-dns-cache -O- -q "$mangapage" > /tmp/input.html
    python3 -c "import sys;import re;from bs4 import BeautifulSoup;pagex=open(\"/tmp/input.html\");pagey=pagex.read();soup=BeautifulSoup(pagey, 'html.parser');alla=soup.find_all(\"a\",text=True,href=True);[sys.stdout.write('https://mangarock.com'+line['href']+'\n') for line in alla if re.search(\"/manga/mrs-serie-.*/chapter/mrs-chapter\", line['href'])]"
    rm -f /tmp/input.html
    deactivate
    }
    export -f mangaissueslist

    mangaissuedownload() {
    cd "${temp_folder}/manga_tmp"
    @@ -369,7 +360,7 @@ for manga in "${manganames[@]}"; do
    awk -v lin="${manga_link}" '{print lin $0;}' "/tmp/manga_lastpdfs" | xargs -l1 -I {} bash -c "manga=\"$manga\"; manga_lastpdfs=\"$manga_lastpdfs\"; temp_folder=\"$temp_folder\"; mangaissuedownload \"\$@\"" _ {}
    sleep 2
    else
    # controlla se nuovo numero
    # check if new issue
    newlist=$(mangaissueslist)
    newlines=$(diff --old-line-format="" --new-line-format="%L" --unchanged-line-format="" <(cat "$manga_lastpdfs") <(echo -e "$newlist"))
    if [[ "$newlines" != "" ]]; then
  16. nestukh created this gist Jun 1, 2019.
    395 changes: 395 additions & 0 deletions manga_checker.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,395 @@
    #!/bin/bash
    ## insert in crontab with
    ## crontab -e
    # 1 * * * * bash /home/USER/manga_checker.sh # every minute 1 of every hour of any day
    ## rememeber to chmod +x manga_checker.sh


    # # # # ### on raspberry pi
    # # # #
    # # # # usermod -a -G sudo,crontab,ssh $USER #### reboot for change to take effect
    # # # #

    # # # #
    # # # # sudo raspi-config ######## expand root to all sd, Wait for Network at Boot, change local time #### reboot for change to take effect
    # # # #

    # # # #
    # # # # echo "dash dash/sh boolean false" | sudo debconf-set-selections
    # # # # sudo dpkg-reconfigure -f noninteractive dash
    # # # # chsh -s /bin/bash # need reboot to take effect
    # # # #

    # # # #
    # # # # echo -e "PKG_CONFIG_PATH=/usr/lib/arm-linux-gnueabihf:\$PKG_CONFIG_PATH
    # # # # export PKG_CONFIG_PATH" >> $HOME/.bashrc
    # # # #
    # # # # sudo apt-get install --reinstall -y resolvconf apt-transport-https ca-certificates
    # # # # # DNS
    # # # # sudo mkdir -p /etc/resolvconf/resolv.conf.d/base
    # # # # echo -e "nameserver 208.67.222.222
    # # # # nameserver 208.67.220.220
    # # # # nameserver 8.8.8.8
    # # # # nameserver 8.8.4.4" | sudo tee /etc/resolvconf/resolv.conf.d/base >> /dev/null
    # # # # sudo resolvconf -u
    # # # # # https://www.raspbian.org/RaspbianMirrors
    # # # # sudo sed -i -e "s/raspbian.raspberrypi.org/mirrordirector.raspbian.org/" /etc/apt/sources.list
    # # # # #sudo sed -i -e "s/http:\/\/raspbian.raspberrypi.org/https:\/\/raspbian.raspberrypi.org/" /etc/apt/sources.list
    # # # # sudo sed -i 's/^#deb-src/deb-src/g' /etc/apt/sources.list
    # # # # sudo sed -i 's/^#deb-src/deb-src/g' /etc/apt/sources.list.d/raspi.list
    # # # # sudo apt-get update && sudo apt-get dist-upgrade -y
    # # # #
    # # # # # mount /tmp on ram in order to reduce sd card wear
    # # # # # echo "tmpfs /tmp tmpfs defaults,noatime 0 0" | sudo tee -a /etc/fstab > /dev/null
    # # # # # echo "tmpfs /var/tmp tmpfs nodev,nosuid 0 0" | sudo tee -a /etc/fstab > /dev/null # ,size=50M #### reboot for change to take effect
    # # # #

    # # # #
    # # # # pypcks="virtualenv python3-pip python3 python3-all-dev python3-dev libffi-dev libssl-dev librtmp-dev python3 python3-setuptools cython build-essential"
    # # # # allgoodpcks="inotify-tools ssh ntpdate unattended-upgrades"
    # # # # sudo apt-get install -y $allgoodpcks $pypcks
    # # # #
    # # # # ### search last commands with up arrow key:
    # # # # ### http://codeinthehole.com/tips/the-most-important-command-line-tip-incremental-history-searching-with-inputrc/
    # # # # echo -e "\"\\\\e[A\": history-search-backward
    # # # # \"\\\\e[B\": history-search-forward
    # # # # \"\\\\e[C\": forward-char
    # # # # \"\\\\e[D\": backward-char" > ~/.inputrc
    # # # #
    # # # # sudo sed -i "s/^session optional pam_mail.so/#session optional pam_mail.so/g" /etc/pam.d/sshd
    # # # # echo "ClientAliveInterval 30" | sudo tee -a /etc/ssh/sshd_config > /dev/null
    # # # # echo "ClientAliveCountMax 5" | sudo tee -a /etc/ssh/sshd_config > /dev/null
    # # # # echo "ServerAliveInterval 15" | sudo tee -a /etc/ssh/ssh_config > /dev/null
    # # # # echo "ServerAliveCountMax 3" | sudo tee -a /etc/ssh/ssh_config > /dev/null
    # # # #
    # # # # # to install locales
    # # # # timezonex="Europe/Rome" # example
    # # # # localex1="en_US.UTF-8"
    # # # # localex2="it_IT.UTF-8" # example
    # # # # echo "$timezonex" | sudo tee /etc/timezone > /dev/null
    # # # # sudo dpkg-reconfigure -f noninteractive tzdata
    # # # # sudo sed -i -e "s/# $localex1 UTF-8/$localex1 UTF-8/" /etc/locale.gen
    # # # # sudo sed -i -e "s/# $localex2 UTF-8/$localex2 UTF-8/" /etc/locale.gen
    # # # # echo "LANG=\"$localex1\"" | sudo tee /etc/default/locale > /dev/null
    # # # # sudo dpkg-reconfigure --frontend=noninteractive locales
    # # # # sudo update-locale LANG=$localex1
    # # # #
    # # # #
    # # # # # time server, updated every boot
    # # # # # The Rasbperry Pi doesn't have a Real Time Clock, so that is why it is resetting itself to 1970 everytime you boot without Internet.
    # # # # echo -e "DAEMONS=(\041hwclock ntpd)" | sudo tee -a /etc/rc.conf >> /dev/null
    # # # # sudo ntpdate -u pool.ntp.org
    # # # #
    # # # # sudo update-ca-certificates --fresh
    # # # #
    # # # #
    # # # # echo -e "APT::Periodic::Update-Package-Lists \"1\";
    # # # # APT::Periodic::Download-Upgradeable-Packages \"1\";
    # # # # APT::Periodic::AutocleanInterval \"7\";
    # # # # APT::Periodic::Unattended-Upgrade \"1\";" | sudo tee -a /etc/apt/apt.conf.d/10periodic > /dev/null
    # # # #
    # # # #
    # # # # # http://www.cl.cam.ac.uk/~atm26/ephemeral/rpi/dwc_otg/doc/html/module%20parameters.html
    # # # # sudo sed -i "s/^vm.min_free_kbytes.*$/vm.min_free_kbytes = 32768/g" /etc/sysctl.conf
    # # # #
    # # # # sudo sed -i "s/smsc95xx.turbo_mode=N//g" /boot/cmdline.txt; sudo sed -i '/rootfstype/ s/$/ smsc95xx.turbo_mode=N/' /boot/cmdline.txt
    # # # # sudo sed -i "/^avoid_safe_mode=.*$/d" /boot/config.txt; echo "avoid_safe_mode=1" | sudo tee -a /boot/config.txt > /dev/null
    # # # #
    # # # #
    # # # #
    # # # # # Syncthing https://syncthing.net/ https://raspberrypi.local:8384
    # # # # curl -s https://syncthing.net/release-key.txt | sudo apt-key add -
    # # # # echo deb http://apt.syncthing.net/ syncthing release | sudo tee /etc/apt/sources.list.d/syncthing-release.list
    # # # # sudo apt-get update
    # # # # sudo apt-get install --reinstall -y syncthing
    # # # #
    # # # # timeout --kill-after=2 -sHUP 4m syncthing
    # # # # echo -e "#\041/bin/sh
    # # # # ### BEGIN INIT INFO
    # # # # # Provides: Syncthing
    # # # # # Required-Start: \$local_fs \$remote_fs \$network
    # # # # # Required-Stop: \$local_fs \$remote_fs \$network
    # # # # # Default-Start: 2 3 4 5
    # # # # # Default-Stop: 0 1 6
    # # # # # Short-Description: Syncthing
    # # # # # Description: Syncthing is for backups
    # # # # ### END INIT INFO
    # # # #
    # # # # # Documentation available at
    # # # # # http://refspecs.linuxfoundation.org/LSB_3.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptfunc.html
    # # # # # Debian provides some extra functions though
    # # # # . /lib/lsb/init-functions
    # # # #
    # # # # DAEMON_NAME=\"syncthing\"
    # # # # DAEMON_USER=$USER
    # # # # DAEMON_PATH=\"/usr/bin/syncthing\"
    # # # # DAEMON_OPTS=\"\"
    # # # # DAEMON_PWD=\"\${PWD}\"
    # # # # DAEMON_DESC=\$(get_lsb_header_val \$0 \"Short-Description\")
    # # # # DAEMON_PID=\"/var/run/\${DAEMON_NAME}.pid\"
    # # # # DAEMON_NICE=0
    # # # # DAEMON_LOG='/var/log/syncthing'
    # # # #
    # # # # [ -r \"/etc/default/\${DAEMON_NAME}\" ] && . \"/etc/default/\${DAEMON_NAME}\"
    # # # #
    # # # # do_start() {
    # # # # local result
    # # # #
    # # # # pidofproc -p \"\${DAEMON_PID}\" \"\${DAEMON_PATH}\" > /dev/null
    # # # # if [ \$? -eq 0 ]; then
    # # # # log_warning_msg \"\${DAEMON_NAME} is already started\"
    # # # # result=0
    # # # # else
    # # # # log_daemon_msg \"Starting \${DAEMON_DESC}\" \"\${DAEMON_NAME}\"
    # # # # touch \"\${DAEMON_LOG}\"
    # # # # chown \$DAEMON_USER \"\${DAEMON_LOG}\"
    # # # # chmod u+rw \"\${DAEMON_LOG}\"
    # # # # if [ -z \"\${DAEMON_USER}\" ]; then
    # # # # start-stop-daemon --start --quiet --oknodo --background \
    # # # # --nicelevel \$DAEMON_NICE \
    # # # # --chdir \"\${DAEMON_PWD}\" \
    # # # # --pidfile \"\${DAEMON_PID}\" --make-pidfile \
    # # # # --exec \"\${DAEMON_PATH}\" -- \$DAEMON_OPTS
    # # # # result=\$?
    # # # # else
    # # # # start-stop-daemon --start --quiet --oknodo --background \
    # # # # --nicelevel \$DAEMON_NICE \
    # # # # --chdir \"\${DAEMON_PWD}\" \
    # # # # --pidfile \"\${DAEMON_PID}\" --make-pidfile \
    # # # # --chuid \"\${DAEMON_USER}\" \
    # # # # --exec \"\${DAEMON_PATH}\" -- \$DAEMON_OPTS
    # # # # result=\$?
    # # # # fi
    # # # # log_end_msg \$result
    # # # # fi
    # # # # return \$result
    # # # # }
    # # # #
    # # # # do_stop() {
    # # # # local result
    # # # #
    # # # # pidofproc -p \"\${DAEMON_PID}\" \"\${DAEMON_PATH}\" > /dev/null
    # # # # if [ \$? -ne 0 ]; then
    # # # # log_warning_msg \"\${DAEMON_NAME} is not started\"
    # # # # result=0
    # # # # else
    # # # # log_daemon_msg \"Stopping \${DAEMON_DESC}\" \"\${DAEMON_NAME}\"
    # # # # killproc -p \"\${DAEMON_PID}\" \"\${DAEMON_PATH}\"
    # # # # result=\$?
    # # # # log_end_msg \$result
    # # # # rm \"\${DAEMON_PID}\"
    # # # # fi
    # # # # return \$result
    # # # # }
    # # # #
    # # # # do_restart() {
    # # # # local result
    # # # # do_stop
    # # # # result=\$?
    # # # # if [ \$result = 0 ]; then
    # # # # do_start
    # # # # result=\$?
    # # # # fi
    # # # # return \$result
    # # # # }
    # # # #
    # # # # do_status() {
    # # # # local result
    # # # # status_of_proc -p \"\${DAEMON_PID}\" \"\${DAEMON_PATH}\" \"\${DAEMON_NAME}\"
    # # # # result=\$?
    # # # # return \$result
    # # # # }
    # # # #
    # # # # do_usage() {
    # # # # echo \$\"Usage: \$0 {start | stop | restart | status}\"
    # # # # exit 1
    # # # # }
    # # # #
    # # # # case \"\$1\" in
    # # # # start) do_start; exit \$? ;;
    # # # # stop) do_stop; exit \$? ;;
    # # # # restart) do_restart; exit \$? ;;
    # # # # status) do_status; exit \$? ;;
    # # # # *) do_usage; exit 1 ;;
    # # # # esac" | sudo tee /etc/init.d/syncthing >> /dev/null
    # # # # sudo chmod +x /etc/init.d/syncthing
    # # # # sudo update-rc.d syncthing defaults
    # # # # sudo service syncthing start
    # # # # sed -i -e "s/tls=\"false\"/tls=\"true\"/g;s/127.0.0.1/0.0.0.0/g" $HOME/.config/syncthing/config.xml
    # # # # sudo service syncthing restart
    # # # #
    # # # #



    export PATH="$PATH:/sbin:/usr/local/sbin:/usr/bin:/usr/local/bin:/usr/sbin:/bin:/usr/local/games:/usr/games:" # not included in crontab env
    source $HOME/.bashrc



    # better to keep log_ and temp_ folders on an usb disk for reducing sd card wear
    log_folder="path/to/your/log/folder"
    temp_folder="path/to/your/temp/folder"



    if [[ ! ( -d "$HOME/bin/comics_dowloander_python_en" ) ]]; then
    python3 -m virtualenv $HOME/bin/comics_dowloander_python_env --python=$(ls -1 /usr/bin/python3* | grep -P "\d$" | tail -n1) # python2 will be decommissioned on January 1, 2020
    source $HOME/bin/comics_dowloander_python_env/bin/activate
    pip install --upgrade pip
    pip install --upgrade --no-cache-dir beautifulsoup4 regex
    deactivate
    fi

    if [[ !(-d "${temp_folder}/manga_tmp") ]]; then
    mkdir -p "${temp_folder}/manga_tmp" # comics-downloader folder
    wget -O "${temp_folder}/manga_tmp/comics-downloader" "$(wget -q -O- https://github.com/Girbons/comics-downloader | grep "releases/download/" | grep -F "$(echo $(uname -m) | sed "s/armv7l/raspian-arm64\"/g;s/armv6l/raspbian-arm\"/g;s/x86_64/linux/g")" | sed "s/^.*href=\"//g;s/\".*$//g")"
    chmod +x "${temp_folder}/manga_tmp/comics-downloader"
    fi


    declare -a manganames
    declare -a mangapages



    ############################################################# MANGA LIST
    ######### #########

    k=0
    manganames[$k]="Hokuto no Ken"
    mangapages[$k]="https://mangarock.com/manga/mrs-serie-107406"

    let "k=k+1"
    manganames[$k]="Vagabond"
    mangapages[$k]="https://mangarock.com/manga/mrs-serie-267124"

    let "k=k+1"
    manganames[$k]="One Punch-Man"
    mangapages[$k]="https://mangarock.com/manga/mrs-serie-358279"



    let "k=k+1"
    manganames[$k]="Nausicaa of the Valley of the Wind"
    mangapages[$k]="https://mangarock.com/manga/mrs-serie-174706"

    let "k=k+1"
    manganames[$k]="Shuna no Tabi"
    mangapages[$k]="https://mangarock.com/manga/mrs-serie-226985"


    let "k=k+1"
    manganames[$k]="Ikigami"
    mangapages[$k]="https://mangarock.com/manga/mrs-serie-114471"


    let "k=k+1"
    manganames[$k]="Battle Angel Alita"
    mangapages[$k]="https://mangarock.com/manga/mrs-serie-90619"
    let "k=k+1"
    manganames[$k]="Ashen Victor"
    mangapages[$k]="https://mangarock.com/manga/mrs-serie-100254952"
    let "k=k+1"
    manganames[$k]="Alita Last Order" # Gunnm: Last Order
    mangapages[$k]="https://mangarock.com/manga/mrs-serie-24827"
    let "k=k+1"
    manganames[$k]="Battle Angel Alita: Holy Night and Other Stories"
    mangapages[$k]="https://mangarock.com/manga/mrs-serie-100334155"
    let "k=k+1"
    manganames[$k]="Battle Angel Alita - Mars Chronicle"
    mangapages[$k]="https://mangarock.com/manga/mrs-serie-90681"

    ######### #########
    ############################################################# MANGA LIST







    q=0
    q1=1
    let "k1=k+1"
    for manga in "${manganames[@]}"; do
    echo "checking $manga"
    manga_lastpdfs="${log_folder}/mangalist_$(echo "${manga}" | sed "s/ /+/g;s/://g")_lastpdfs"
    echo "$manga_lastpdfs"
    echo "${mangapages[$q]}"
    mangapage="${mangapages[$q]}"

    mangaissueslist() {
    source $HOME/bin/comics_dowloander_python_env/bin/activate
    wget --tries=256 --timeout=256 --wait=$(( ( RANDOM % 10 ) + 1 )) --waitretry=$(( ( RANDOM % 10 ) + 1 )) --random-wait --no-dns-cache -O- -q "$mangapage" > /tmp/input.html
    python3 -c "import sys;import re;from bs4 import BeautifulSoup;pagex=open(\"/tmp/input.html\");pagey=pagex.read();soup=BeautifulSoup(pagey, 'html.parser');alla=soup.find_all(\"a\",text=True,href=True);[sys.stdout.write('https://mangarock.com'+line['href']+'\n') for line in alla if re.search(\"/manga/mrs-serie-.*/chapter/mrs-chapter\", line['href'])]"
    rm -f /tmp/input.html
    deactivate
    }
    export -f mangaissueslist

    mangapagedownload() {
    source $HOME/bin/comics_dowloander_python_env/bin/activate
    wget --tries=256 --timeout=256 --wait=$(( ( RANDOM % 10 ) + 1 )) --waitretry=$(( ( RANDOM % 10 ) + 1 )) --random-wait --no-dns-cache -O- -q "$mangapage" > /tmp/input.html
    python3 -c "import sys;import re;from bs4 import BeautifulSoup;pagex=open(\"/tmp/input.html\");pagey=pagex.read();soup=BeautifulSoup(pagey, 'html.parser');alla=soup.find_all(\"a\",text=True,href=True);[sys.stdout.write('https://mangarock.com'+line['href']+'\n') for line in alla if re.search(\"/manga/mrs-serie-.*/chapter/mrs-chapter\", line['href'])]"
    rm -f /tmp/input.html
    deactivate
    }
    export -f mangaissueslist

    mangaissuedownload() {
    cd "${temp_folder}/manga_tmp"
    j="$(cat "/tmp/manga_incrj")"
    filex=""
    while [[ "$filex" == "" ]]; do # to prevent errors
    echo "[$j/$totalj/$q1/$k1]: dowloading issue [$j] of [$totalj] on link $1 for $manga [$q1] (total number of mangas to download: [$k1])"
    ./comics-downloader -url="$1" -format=pdf
    sleep 3
    filex="$(find $PWD -mindepth 1 -type f -name "*.pdf" -mmin -$((6*60)) -exec ls {} \; | head -n1)"
    done
    # mv "$filex" "${SHARED_SYNCTHING_FOLDER}/$manga - $(basename "${filex%.*}").pdf" ######## this line move the pdf file to the shared synchting folder, or dropbox folder or else
    echo "$1" >> "$manga_lastpdfs"
    let "j=j+1";
    echo "$j" > "/tmp/manga_incrj"
    }
    export -f mangaissuedownload


    if [[ ! ( -f "$manga_lastpdfs" ) ]]; then
    touch "$manga_lastpdfs"
    rm -f "/tmp/manga_lastpdfs"
    mangaissueslist > "/tmp/manga_lastpdfs"
    unset manga_link
    j=1
    echo "$j" > "/tmp/manga_incrj"
    totalj="$(cat "/tmp/manga_lastpdfs" | wc -l)"
    export totalj
    export q1
    export k1
    awk -v lin="${manga_link}" '{print lin $0;}' "/tmp/manga_lastpdfs" | xargs -l1 -I {} bash -c "manga=\"$manga\"; manga_lastpdfs=\"$manga_lastpdfs\"; temp_folder=\"$temp_folder\"; mangaissuedownload \"\$@\"" _ {}
    sleep 2
    else
    # controlla se nuovo numero
    newlist=$(mangaissueslist)
    newlines=$(diff --old-line-format="" --new-line-format="%L" --unchanged-line-format="" <(cat "$manga_lastpdfs") <(echo -e "$newlist"))
    if [[ "$newlines" != "" ]]; then
    echo "new issue for this manga!"
    unset manga_link
    j=1
    echo "$j" > "/tmp/manga_incrj"
    totalj="$(echo -e "$newlines" | wc -l)"
    export totalj
    export q1
    export k1
    echo -e "$newlines" | awk -v lin="${manga_link}" '{print lin $0;}' | xargs -l1 -I {} bash -c "manga=\"$manga\"; manga_lastpdfs=\"$manga_lastpdfs\"; temp_folder=\"$temp_folder\"; mangaissuedownload \"\$@\"" _ {}
    sleep 2
    else
    echo "no new issue for this manga."
    fi
    fi
    echo
    echo
    let "q=q+1"
    let "q1=q+1"
    done