onceupon · September 15, 2017 03:09 · Sep 15, 2017 · Sep 15, 2017
diff --git a/Bash-Oneliner → bash-oneliner.sh b/Bash-Oneliner → bash-oneliner.sh
diff --git a/Bash-Oneliner b/Bash-Oneliner
@@ -0,0 +1,1979 @@
+# ### Bash-Oneliner
+# Hi bash learners and bioinformaticans, welcome to Bash Oneliner. I started studying bioinformatics data three years ago (recently started working on cloud computing), and was amazed by those single-word bash commands which are much faster than my dull scripts, so i started bash. Not all the code here is oneliner (if the ';' counts..), but i put effort on making them brief and fast. 
+
+# This blog will focus on simple bash commands for parsing data, most of which are for tsv files (tab-separated values); some of them are for Linux system maintenance. I apologize that there won't be any citation for the codes, but they are probably from dear Google and Stackoverflow.
+
+# English and bash are not my first language, so... correct me anytime, thank you
+
+# In case you would like to check and vote up my questions on Stackoverflow, here's my page:
+# http://stackoverflow.com/users/4290753/once
+
+# Here's a more stylish version of Bash-Oneliner~
+# http://onceupon.github.io/Bash-Oneliner/
+
+# ## Handy Bash oneliner commands for tsv file editing
+
+# - [Grep](#grep)
+# - [Sed](#sed)
+# - [Awk](#awk)
+# - [Xargs](#xargs)
+# - [Find](#find)
+# - [Loops](#loops)
+# - [Math](#math)
+# - [Download](#download)
+# - [Random](#random)
+# - [Xwindow](#xwindow)
+# - [Others](#others)
+# - [System](#system)
+
+## Grep
+#####  Extract text bewteen words (e.g. w1,w2)
+
+
+grep -o -P '(?<=w1).*(?=w2)'
+
+
+##### Grep lines without word (e.g. bbo)
+
+
+grep -v bbo filename
+
+
+##### Grep only one/first match (e.g. bbo)
+
+
+grep -m 1 bbo filename
+
+
+##### Grep and count (e.g. bbo)
+
+
+grep -c bbo filename
+
+
+##### Insensitive grep (e.g. bbo/BBO/Bbo)
+
+
+grep -i "bbo" filename 
+
+
+##### Count occurrence (e.g. three times a line count three times)
+
+
+grep -o bbo filename 
+
+
+##### COLOR the match (e.g. bbo)!
+
+
+grep --color bbo filename 
+
+
+##### Grep search all files in a directory(e.g. bbo)
+
+
+grep -R bbo /path/to/directory 
+
+
+# or
+
+
+grep -r bbo /path/to/directory 
+
+##### Search all files in directory, only output file names with matches(e.g. bbo)
+
+
+grep -Rh bbo /path/to/directory 
+
+# or
+
+grep -rh bbo /path/to/directory 
+
+
+##### Grep OR (e.g. A or B or C or D)
+
+
+grep 'A\|B\|C\|D' 
+
+
+##### Grep AND (e.g. A and B)
+
+
+grep 'A.*B' 
+
+
+##### Grep all content of a fileA from fileB
+
+
+grep -f fileA fileB 
+
+
+##### Grep a tab
+
+
+grep $'\t' 
+
+
+##### Grep variable from variable
+
+$echo "$long_str"|grep -q "$short_str"
+if [ $? -eq 0 ]; then echo 'found'; fi
+
+# grep -q will output 0 if match found  
+# remember to add space between []!
+
+##### Grep strings between a bracket()
+
+grep -oP '\(\K[^\)]+'
+
+
+##### Grep number of characters with known strings in between(e.g. AAEL000001-RA)
+
+grep -o -w "\w\{10\}\-R\w\{1\}"
+
+# \w word character [0-9a-zA-Z_] \W not word character 
+
+
+##### A lot examples here: http://www.cyberciti.biz/faq/grep-regular-expressions/
+
+
+
+## Sed
+##### Remove lines with word (e.g. bbo)
+
+
+sed "/bbo/d" filename
+
+
+##### Edit infile (edit and save)
+
+
+sed -i "/bbo/d" filename
+
+##### When using variable (e.g. $i), use double quotes " "
+# e.g. add >$i to the first line (to make a FASTA file)  
+
+
+sed "1i >$i"  
+
+# notice the double quotes! in other examples, you can use a single quote, but here, no way!   
+# '1i' means insert to first line
+
+
+##### Delete empty lines
+
+
+sed '/^\s*$/d' 
+
+# or
+
+
+sed 's/^$/d' 
+
+##### Delete last line
+
+
+sed '$d' 
+
+
+##### Delete last character from end of file
+
+sed -i '$ s/.$//' filename
+
+
+##### Add string to end of file (e.g. "]")
+
+
+sed '$s/$/]/' filename
+
+##### Add newline to the end
+
+
+sed '$a\'
+
+
+##### Add string to beginning of every line (e.g. bbo)
+
+
+sed -e 's/^/bbo/' file
+
+
+##### Add string to end of each line (e.g. "}")
+
+sed -e 's/$/\}\]/' filename
+
+
+##### Add \n every nth character (e.g. every 4th character)
+
+
+sed 's/.\{4\}/&\n/g' 
+
+
+##### Concatenate/combine/join files with a seperator and next line (e.g seperate by ",")
+
+
+sed -s '$a,' *.json > all.json
+
+
+##### Substitution (e.g. replace A by B)
+
+
+sed 's/A/B/g' filename 
+
+##### Select lines start with string (e.g. bbo)
+
+
+sed -n '/^@S/p' 
+
+##### Delete lines with string (e.g. bbo)
+
+
+sed '/bbo/d' filename 
+
+##### Print every nth lines
+
+
+sed -n '0~3p' filename
+
+# catch 0: start; 3: step
+
+
+##### Print every odd # lines
+
+
+sed -n '1~2p' 
+
+##### Print every third line including the first line
+
+
+sed -n '1p;0~3p' 
+
+##### Remove leading whitespace and tabs
+
+
+sed -e 's/^[ \t]*//'
+
+# notice a whitespace before '\t'!!
+
+
+##### Remove only leading whitespace
+
+
+sed 's/ *//'
+
+# notice a whitespace before '*'!!
+
+
+##### Remove ending commas
+
+
+sed 's/,$//g' 
+
+##### Add a column to the end
+
+
+sed "s/$/\t$i/"
+
+# $i is the valuable you want to add  
+# e.g. add the filename to every last column of the file  
+
+
+for i in $(ls);do sed -i "s/$/\t$i/" $i;done
+
+##### Add extension of filename to last column
+
+
+for i in T000086_1.02.n T000086_1.02.p;do sed "s/$/\t${i/*./}/" $i;done >T000086_1.02.np
+
+
+##### Remove newline\ nextline
+
+
+sed ':a;N;$!ba;s/\n//g'
+
+
+##### Print a number of lines (e.g. line 10th to line 33 rd)
+
+
+sed -n '10,33p' <filename
+
+
+##### Change delimiter 
+
+sed 's=/=\\/=g'
+
+
+##### Replace with wildcard (e.g A-1-e or A-2-e or A-3-e....)
+
+sed 's/A-.*-e//g' filename
+
+##### Remove last character of file
+
+sed '$ s/.$//'
+
+
+##### Insert character at specified position of file (e.g. AAAAAA --> AAA#AAA)
+
+sed -r -e 's/^.{3}/&#/' file
+
+
+
+## Awk
+
+##### Set tab as field separator
+
+
+awk -F $'\t'  
+
+
+##### Output as tab separated (also as field separator)
+
+
+awk -v OFS='\t' 
+
+
+##### Pass variable
+
+
+a=bbo;b=obb;
+awk -v a="$a" -v b="$b" "$1==a && $10=b" filename
+
+
+##### Print line number and number of characters on each line
+
+
+awk '{print NR,length($0);}' filename 
+
+
+##### Find number of columns
+
+
+awk '{print NF}' 
+
+
+##### Reverse column order
+
+
+awk '{print $2, $1}' 
+
+
+##### Check if there is a comma in a column (e.g. column $1)
+
+
+awk '$1~/,/ {print}'  
+
+
+##### Split and do for loop
+
+
+awk '{split($2, a,",");for (i in a) print $1"\t"a[i]}' filename 
+
+
+##### Print all lines before nth occurence of a string (e.g stop print lines when bbo appears 7 times)
+
+
+awk -v N=7 '{print}/bbo/&& --N<=0 {exit}'
+
+
+##### Print filename and last line of all files in directory
+
+ls|xargs -n1 -I file awk '{s=$0};END{print FILENAME,s}' file
+
+
+##### Add string to the beginning of a column (e.g add "chr" to column $3)
+
+
+awk 'BEGIN{OFS="\t"}$3="chr"$3' 
+
+
+##### Remove lines with string (e.g. bbo)
+
+
+awk '!/bbo/' file 
+
+
+##### Column subtraction
+
+
+cat file| awk -F '\t' 'BEGIN {SUM=0}{SUM+=$3-$2}END{print SUM}'
+
+
+##### Usage and meaning of NR and FNR
+# e.g.  
+# fileA:  
+# a  
+# b  
+# c  
+# fileB:  
+# d  
+# e  
+
+
+awk 'print FILENAME, NR,FNR,$0}' fileA fileB 
+
+
+# fileA    1    1    a  
+# fileA    2    2    b  
+# fileA    3    3    c  
+# fileB    4    1    d  
+# fileB    5    2    e  
+
+##### AND gate
+
+# e.g.  
+# fileA:  
+# 1    0  
+# 2    1  
+# 3    1  
+# 4    0  
+
+# fileB:  
+# 1    0  
+# 2    1  
+# 3    0  
+# 4    1  
+
+
+awk -v OFS='\t' 'NR=FNR{a[$1]=$2;next} NF {print $1,((a[$1]=$2)? $2:"0")}' fileA fileB 
+
+
+# 1    0  
+# 2    1  
+# 3    0  
+# 4    0  
+
+##### Round all numbers of file (e.g. 2 significant figure)
+
+
+awk '{while (match($0, /[0-9]+\[0-9]+/)){
+    \printf "%s%.2f", substr($0,0,RSTART-1),substr($0,RSTART,RLENGTH)
+    \$0=substr($0, RSTART+RLENGTH)
+    \}
+    \print
+    \}'
+
+
+##### Give number/index to every row
+
+
+awk '{printf("%s\t%s\n",NR,$0)}'
+
+##### Break combine column data into rows
+
+# e.g.  
+# seperate  
+
+# David    cat,dog  
+# into  
+# David    cat  
+# David    dog  
+
+# detail here:　http://stackoverflow.com/questions/33408762/bash-turning-single-comma-separated-column-into-multi-line-string
+
+awk '{split($2,a,",");for(i in a)print $1"\t"a[i]}' file
+
+
+##### Sum up a file (each line in file contains only one number)
+
+
+awk '{s+=$1} END {print s}' filename
+
+
+##### Average a file (each line in file contains only one number)
+
+awk '{s+=$1}END{print s/NR}'
+
+
+##### Print field start with string (e.g Linux)
+
+
+awk '$1 ~ /^Linux/'
+
+
+##### Sort a row (e.g. 1 40  35  12  23  --> 1 12    23  35  40)
+
+
+awk ' {split( $0, a, "\t" ); asort( a ); for( i = 1; i <= length(a); i++ ) printf( "%s\t", a[i] ); printf( "\n" ); }'
+
+
+##### Subtract previous row values (add column6 which equal to column4 minus last column5) 
+
+awk '{$6 = $4 - prev5; prev5 = $5; print;}'
+
+
+
+
+## Xargs
+
+
+##### Set tab as delimiter (default:space)
+
+
+xargs -d\t
+
+
+##### Display 3 items per line
+
+
+echo 1 2 3 4 5 6| xargs -n 3
+
+
+# 1 2 3  
+# 4 5 6
+
+
+##### Prompt before execution
+
+
+echo a b c |xargs -p -n 3
+
+
+##### Print command along with output
+
+
+xargs -t abcd
+
+
+# /bin/echo abcd  
+# abcd
+
+
+##### With find and rm
+
+
+find . -name "*.html"|xargs rm -rf
+
+
+##### Delete fiels with whitespace in filename (e.g. "hello 2001")
+
+
+find . -name "*.c" -print0|xargs -0 rm -rf
+
+
+##### Show limits
+
+
+xargs --show-limits
+
+
+##### Move files to folder
+
+
+find . -name "*.bak" -print 0|xargs -0 -I {} mv {} ~/old
+
+
+or
+
+
+find . -name "*.bak" -print 0|xargs -0 -I file mv file ~/old
+
+
+##### Move first 100th files to a directory (e.g. d1)
+
+
+ls |head -100|xargs -I {} mv {} d1
+
+
+##### Parallel
+
+
+time echo {1..5} |xargs -n 1 -P 5 sleep
+
+# a lot faster than
+
+time echo {1..5} |xargs -n1 sleep
+
+
+##### Copy all files from A to B
+
+
+find /dir/to/A -type f -name "*.py" -print 0| xargs -0 -r -I file cp -v -p file --target-directory=/path/to/B
+
+
+# v: verbose|  
+# p: keep detail (e.g. owner)
+
+
+##### With sed
+
+
+ls |xargs -n1 -I file sed -i '/^Pos/d' filename
+
+
+##### Add the file name to the first line of file
+
+
+ls |sed 's/.txt//g'|xargs -n1 -I file sed -i -e '1 i\>file\' file.txt
+
+
+##### Count all files
+
+
+ls |xargs -n1 wc -l
+
+
+##### Turn output into a single line
+
+
+ls -l| xargs
+
+
+##### Count files within directories
+
+
+echo mso{1..8}|xargs -n1 bash -c 'echo -n "$1:"; ls -la "$1"| grep -w 74 |wc -l' --
+
+ # "--" signals the end of options and display further option processing
+
+
+##### Download dependencies files and install (e.g. requirements.txt)
+
+
+cat requirements.txt| xargs -n1 sudo pip install
+
+
+##### Count lines in all file, also count total lines
+
+
+ls|xargs wc -l
+
+##### Xargs and grep
+
+
+cat grep_list |xargs -I{} grep {} filename
+
+
+## Find 
+
+##### List all sub directory/file in the current directory
+
+
+find .
+
+
+##### List all files under the current directory
+
+
+find . -type f
+
+
+##### List all directories under the current directory
+
+
+find . -type d
+
+
+##### Edit all files under current directory (e.g. replace 'www' with 'ww')
+
+
+find . name '*.php' -exec sed -i 's/www/w/g' {} \;
+
+# if no subdirectory
+
+
+replace "www" "w" -- *
+
+# a space before *
+
+
+##### Find and output only filename (e.g. "mso")
+
+
+find mso*/ -name M* -printf "%f\n"
+
+
+##### Find and delete file with size less than (e.g. 74 byte)
+
+
+find . -name "*.mso" -size -74c -delete
+
+# M for MB, etc
+
+
+## Loops
+
+##### While loop, column subtraction of a file (e.g. a 3 columns file)
+
+
+while read a b c; do echo $(($c-$b));done < <(head filename)
+
+# there is a space between the two '<'s
+
+##### While loop, sum up column subtraction
+
+
+i=0; while read a b c; do ((i+=$c-$b)); echo $i; done < <(head filename)
+
+
+##### If loop
+
+
+if (($j==$u+2))
+
+# (( )) use for arithmetic operation
+
+
+if [[$age >21]]
+
+# [[ ]] use for comparison
+
+
+##### Test if file exist
+
+if [ -e 'filename' ]
+then
+  echo -e "file exists!"
+fi
+
+
+
+##### For loop
+
+
+for i in $(ls); do echo file $i;done
+
+
+## Math
+
+##### Print out the prime factors of a number (e.g. 50)
+
+factor 50
+
+
+##### Simple math with expr
+
+expr 10+20 #30
+expr 10\*20 #600
+expr 30 \> 20 #1 (true)
+
+
+##### More math with bc
+
+# - Number of decimal digit/ significant figure  
+
+echo "scale=2;2/3" | bc  
+#.66
+
+# - Exponent operator  
+
+echo "10^2" | bc  
+#100
+
+
+# - Using variables   
+
+echo "var=5;--var"| bc  
+#4
+
+
+
+
+## Download
+
+##### Download all from a page
+
+
+wget -r -l1 -H -t1 -nd -N -np -A mp3 -e robots=off http://example.com
+
+# -r: recursive and download all links on page  
+# -l1: only one level link  
+# -H: span host, visit other hosts  
+# -t1: numbers of retries  
+# -nd: don't make new directories, download to here  
+# -N: turn on timestamp  
+# -nd: no parent  
+# -A: type (seperate by ,)  
+# -e robots=off: ignore the robots.txt file which stop wget from crashing the site, sorry example.com  
+
+##### Upload a file to web and download (https://transfer.sh/)
+# --> upload:
+
+curl --upload-file ./filename.txt https://transfer.sh/filename.txt
+
+# (the above command will return a URL, e.g: https://transfer.sh/tG8rM/filename.txt)  
+# --> download:
+
+curl https://transfer.sh/tG8rM/filename.txt -o filename.txt
+
+
+
+##### Download file if necessary
+
+
+data=file.txt
+url=http://www.example.com/$data
+if [! -s $data];then
+    echo "downloading test data..."
+    wget $url
+fi
+
+
+##### Wget to a filename (when a long name)
+
+
+wget -O filename "http://example.com"
+
+
+##### Wget files to a folder
+
+
+wget -P /path/to/directory "http://example.com"
+
+
+
+## Random
+##### Random pick 100 lines from a file
+
+
+shuf -n 100 filename
+
+
+##### Random order (lucky draw)
+
+
+for i in a b c d e; do echo $i; done| shuf
+
+
+##### Echo series of random numbers between a range (e.g. shuffle numbers from 0-100, then pick 15 of them randomly)
+
+
+shuf -i 0-100 -n 15
+
+
+##### Echo a random number
+
+
+echo $RANDOM
+
+
+##### Random from 0-9
+
+
+echo $((RANDOM % 10))
+
+
+##### Random from 1-10
+
+
+echo $(((RANDOM %10)+1))
+
+
+## Xwindow
+# X11 GUI applications! Here are some GUI tools for you if you get bored by the text-only environment.  
+
+
+##### Enable X11 forwarding,in order to use graphical application on servers
+
+ssh -X user_name@ip_address
+
+# or setting through xhost
+
+# --> Install the following for Centos:  
+# xorg-x11-xauth  
+# xorg-x11-fonts-*  
+# xorg-x11-utils  
+
+##### Little xwindow tools
+
+
+xclock
+xeyes
+xcowsay
+
+
+##### Open pictures/images from ssh server
+
+# 1. ssh -X user_name@ip_address
+# 2. apt-get install eog
+# 3. eog picture.png
+
+
+##### Use gedit on server (GUI editor) 
+
+# 1. ssh -X user_name@ip_address
+# 2. apt-get install gedit
+# 3. gedit filename.txt
+
+##### Open PDF file from ssh server 
+
+# 1. ssh -X user_name@ip_address
+# 2. apt-get install evince
+# 3. evince filename.pdf
+
+##### Use google-chrome browser from ssh server 
+
+# 1. ssh -X user_name@ip_address
+# 2. apt-get install libxss1 libappindicator1 libindicator7
+# 3. wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+# 4. sudo apt-get install -f
+# 5. dpkg -i google-chrome*.deb
+# 6. google-chrome
+
+
+
+
+
+## Others
+
+##### Remove newline / nextline
+
+
+tr --delete '\n' <input.txt >output.txt
+
+
+##### Replace newline
+
+
+tr '\n' ' ' <filename
+
+
+##### To uppercase/lowercase
+
+
+tr /a-z/ /A-Z/
+
+
+##### Compare files (e.g. fileA, fileB)
+
+
+diff fileA fileB
+
+# a: added; d:delete; c:changed
+
+# or
+
+
+sdiff fileA fileB
+
+# side-to-side merge of file differences
+
+
+##### Number a file (e.g. fileA)
+
+
+nl fileA
+
+# or
+
+
+nl -nrz fileA
+
+# add leading zeros
+
+
+##### Combine/ paste two files (e.g. fileA, fileB)
+
+
+paste fileA fileB
+
+# default tab seperated
+
+
+##### Reverse string
+
+
+echo 12345| rev
+
+
+##### Read .gz file without extracting
+
+
+zmore filename
+
+# or
+
+
+zless filename
+
+
+##### Run in background, output error file
+
+
+seq 100  &>log &
+
+
+# or
+
+
+
+seq 100 2>log &
+
+
+# or
+
+
+seq 100 2>&1| tee logfile
+
+
+# or
+
+
+seq 100 2>&1 >>outfile
+
+# 0: standard input; 1: standard output; 2: standard error
+
+
+##### Send mail
+
+
+echo 'heres the content'| mail -A 'file.txt' -s 'mail.subject' [email protected]
+
+# use -a flag to set send from (-a "From: [email protected]")
+
+
+##### .xls to csv
+
+
+xls2csv filename
+
+##### Append to file (e.g. hihi)
+
+
+echo 'hihi' >>filename
+
+
+##### Make BEEP sound
+
+
+speaker-test -t sine -f 1000 -l1
+
+
+##### Set beep duration
+
+
+(speaker-test -t sine -f 1000) & pid=$!;sleep 0.1s;kill -9 $pid
+
+
+##### History edit/ delete
+
+
+~/.bash_history
+
+# or
+
+
+history -d [line_number]
+
+
+##### Get last history/record filename
+
+
+head !$
+
+
+##### Clean screen
+
+
+clear
+
+
+# or
+
+
+Ctrl+l
+
+
+##### Send data to last edited file
+
+
+cat /directory/to/file
+echo 100>!$
+
+
+##### Run history number (e.g. 53)
+
+
+!53
+
+
+##### Run last command
+
+
+!!
+
+
+##### Run last command that began with (e.g. cat filename)
+
+
+!cat
+
+
+or
+
+
+!c
+
+# run cat filename again
+
+
+##### Extract .xf
+
+    1.unxz filename.tar.xz  
+    2.tar -xf filename.tar
+
+##### Install python package
+
+
+pip install packagename
+
+
+##### Delete current bash command
+
+
+Ctrl+U
+
+
+# or
+
+
+Ctrl+C
+
+
+# or
+
+
+Alt+Shift+#
+
+# to make it to history
+
+
+##### Add something to history (e.g. "addmetohistory")
+
+
+#addmetodistory
+
+# just add a "#" before~~
+
+
+##### Sleep awhile or wait for a moment or schedule a job
+
+
+sleep 5;echo hi
+
+
+##### Count the time for executing a command
+
+
+time echo hi
+
+
+##### Backup with rsync
+
+
+rsync -av filename filename.bak
+rsync -av directory directory.bak
+rsync -av --ignore_existing directory/ directory.bak
+rsync -av --update directory directory.bak
+rsync -av directory user@ip_address:/path/to/directory.bak
+
+# skip files that are newer on receiver (i prefer this one!)
+
+
+##### Make all directories at one time!
+
+
+mkdir -p project/{lib/ext,bin,src,doc/{html,info,pdf},demo/stat}
+
+# -p: make parent directory  
+# this will create project/doc/html/; project/doc/info; project/lib/ext ,etc
+
+
+##### Run command only if another command returns zero exit status (well done)
+
+
+cd tmp/ && tar xvf ~/a.tar
+
+
+##### Run command only if another command returns non-zero exit status (not finish)
+
+
+cd tmp/a/b/c ||mkdir -p tmp/a/b/c
+
+
+##### Extract to a path
+
+
+tar xvf -C /path/to/directory filename.gz
+
+
+##### Use backslash "\" to break long command
+
+
+cd tmp/a/b/c \
+> || \
+>mkdir -p tmp/a/b/c
+
+
+##### Get pwd
+
+
+VAR=$PWD; cd ~; tar xvf -C $VAR file.tar
+
+# PWD need to be capital letter
+
+##### List file type of file (e.g. /tmp/)
+
+
+file /tmp/
+
+# tmp/: directory
+
+
+##### Bash script
+
+
+#!/bin/bash
+file=${1#*.}
+
+# remove string before a "."
+
+
+file=${1%.*}
+
+# remove string after a "."
+
+##### Search from history
+
+
+Ctrl+r
+
+
+##### Python simple HTTP Server
+
+
+python -m SimpleHTTPServer
+
+
+##### Variables
+
+
+{i/a/,}
+
+# e.g. replace all
+
+{i//a/,}
+
+# for variable i, replace all 'a' with a comma
+
+##### Read user input
+
+
+read input
+echo $input
+
+
+##### Generate sequence 1-10
+
+
+seq 10
+
+
+##### Sum up input list (e.g. seq 10)
+
+
+seq 10|paste -sd+|bc
+
+
+##### Find average of input list/file
+
+
+i=`wc -l filename|cut -d ' ' -f1`; cat filename| echo "scale=2;(`paste -sd+`)/"$i|bc
+
+
+##### Generate all combination (e.g. 1,2)
+
+
+echo {1,2}{1,2}
+
+# 1 1, 1 2, 2 1, 2 2
+
+##### Generate all combination (e.g. A,T,C,G)
+
+
+set = {A,T,C,G}
+group= 5
+for ((i=0; i<$group; i++));do
+    repetition=$set$repetition;done
+    bash -c "echo "$repetition""
+
+
+##### Read file content to variable
+
+foo=$(<test1)
+
+
+##### Echo size of variable
+
+
+echo ${#foo}
+
+
+##### Echo tab
+
+
+echo -e ' \t '
+
+
+##### Array
+
+declare -A array=()
+
+
+##### Send a directory
+
+
+scp -r directoryname user@ip:/path/to/send
+
+
+##### Split file into lines (e.g. 1000 lines/smallfile)
+
+split -d -l 1000 bigfilename
+
+
+##### Rename all files (e.g. remove ABC from all .gz files)
+
+rename 's/ABC//' *.gz
+
+
+##### Remove extention (e.g remove .gz from filename.gz)
+
+basename filename.gz .gz
+
+zcat filename.gz> $(basename filename.gz .gz).unpacked
+
+
+##### Use the squeeze repeat option (e.g. /t/t --> /t)
+
+tr -s "/t" < filename
+
+
+##### Do not print nextline with echo
+
+echo -e 'text here \c'
+
+
+##### Use the last argument
+
+!$
+
+
+##### Check last exit code
+
+echo $?
+
+##### View first 50 characters of file
+
+head -c 50 file
+
+
+##### Group/combine rows into one row 
+
+# e.g.  
+# AAAA  
+# BBBB  
+# CCCC  
+# DDDD  
+
+cat filename|paste - -
+# -->
+# AAAABBBB
+# CCCCDDDD
+cat filename|paste - - - -
+# -->
+# AAAABBBBCCCCDDDD
+
+
+##### Fastq to fasta 
+
+cat file.fastq | paste - - - - | sed 's/^@/>/g'| cut -f1-2 | tr '\t' '\n' >file.fa
+
+##### Cut and get last column
+
+cat file|rev | cut -d/ -f1 | rev
+
+
+##### Add one to variable/increment a numeric variable (e.g. $var)
+
+((var++))
+
+
+##### Some handy environment variables
+# $0   :name of shell or shell script.  
+# $1, $2, $3, ... :positional parameters.  
+# $#   :number of positional parameters.  
+# $?   :most recent foreground pipeline exit status.  
+# $-   :current options set for the shell.  
+# $$   :pid of the current shell (not subshell).  
+# $!   :is the PID of the most recent background command.  
+
+##### Clear the contents of a file (e.g. filename)
+
+>filename
+
+##### Unzip tar.bz2 file (e.g. file.tar.bz2)
+
+tar xvfj file.tar.bz2
+
+##### Output a y/n repeatedly until killed 
+# 'y':
+
+yes
+
+# or 'n':
+
+yes n
+
+# or 'anything':
+
+yes anything
+
+
+# For example: 
+
+yes | rm -r large_directory
+
+
+##### Create dummy file of certain size instantly (e.g. 200mb)
+
+dd if=/dev/zero of=//dev/shm/200m bs=1024k count=200
+
+
+# Standard output:  
+# 200+0 records in  
+# 200+0 records out  
+# 209715200 bytes (210 MB) copied, 0.0955679 s, 2.2 GB/s  
+
+##### Cat to a file
+
+cat >myfile
+let me add sth here
+exit by control + c
+^C
+
+
+##### Keep /repeatedly executing the same command (e.g Repeat 'wc -l filename' every 1 second)
+
+watch -n 1 wc -l filename
+
+
+##### Print commands and their arguments when execute (e.g. echo `expr 10 + 20 `)
+
+set -x; echo `expr 10 + 20 `
+
+
+##### Print some meaningful sentences to you (install fortune first)
+
+fortune
+
+##### Colorful (and useful) version of top (install htop first)
+
+htop
+
+
+## System
+
+##### Snapshot of the current processes
+
+
+ps 
+
+
+##### Check graphics card
+
+
+lspci
+
+
+##### Show IP address
+
+
+$ip add show
+
+or
+
+
+ifconfig
+
+
+##### Check system version
+
+
+cat /etc/*-release
+
+
+##### Linux Programmer's Manuel: hier- description of the filesystem hierarchy
+
+
+man hier
+
+
+##### List job
+
+
+jobs -l
+
+
+##### Export PATH
+
+
+export PATH=$PATH:~/path/you/want
+
+
+##### Make file execuable
+
+
+chmod +x filename
+
+# you can now ./filename to execute it
+
+##### List screen
+
+
+screen -d -r
+
+
+##### Echo screen name
+
+
+screen -ls
+
+
+##### Check system (x86-64)
+
+
+uname -i
+
+
+##### Surf the net
+
+
+links www.google.com
+
+
+##### Add user, set passwd
+
+
+useradd username
+passwd username
+
+
+##### Edit variable for bash, (e.g. displaying the whole path)
+
+
+# 1. joe ~/.bash_profile 
+# 2. export PS1='\u@\h:\w\$' 
+
+# $PS1 is a variable that defines the makeup and style of the command prompt 
+
+# 3. source ~/.bash_profile
+
+
+##### Edit environment setting (e.g. alias)
+
+
+# 1. joe ~/.bash_profile
+# 2. alias pd="pwd" //no more need to type that 'w'!
+# 3. source ~/.bash_profile
+
+
+##### List environment variables (e.g. PATH)
+
+
+$echo $PATH
+
+# list of directories separated by a colon
+
+##### List all environment variables for current user
+
+
+$env
+
+
+##### Show partition format
+
+
+lsblk
+
+
+##### Soft link program to bin
+
+
+ln -s /path/to/program /home/usr/bin
+
+# must be the whole path to the program
+
+##### Show hexadecimal view of data
+
+
+hexdump -C filename.class
+
+
+##### Jump to different node
+
+
+rsh node_name
+
+
+##### Check port (active internet connection)
+
+
+netstat -tulpn
+
+
+##### Find whick link to a file
+
+
+readlink filename
+
+
+##### Check where a command link to (e.g. python)
+
+
+which python
+
+
+##### List total size of a directory
+
+
+du -hs .
+
+or
+
+
+du -sb
+
+
+##### Copy directory with permission setting
+
+
+cp -rp /path/to/directory
+
+
+##### Store current directory
+
+
+pushd . $popd ;dirs -l 
+
+
+##### Show disk usage
+
+
+df -h 
+
+# or
+
+
+du -h 
+
+# or
+
+
+du -sk /var/log/* |sort -rn |head -10
+
+
+##### Show current runlevel
+
+
+runlevel
+
+
+##### Switch runlevel
+
+
+init 3 
+
+# or
+
+telinit 3 
+
+##### Permanently modify runlevel
+
+
+1. edit /etc/init/rc-sysinit.conf 
+2. env DEFAULT_RUNLEVEL=2 
+
+
+##### Become root
+
+
+su
+
+
+##### Become somebody
+
+
+su somebody
+
+
+##### Report user quotes on device
+
+
+requota -auvs
+
+
+##### Get entries in a number of important databases
+
+
+getent database_name
+
+# (e.g. the 'passwd' database)
+
+
+getent passwd
+
+# list all user account (all local and LDAP)  
+# (e.g. fetch list of grop accounts)
+
+
+getent group
+
+# store in database 'group'
+
+##### Change owner of file
+
+
+chown user_name filename
+chown -R user_name /path/to/directory/
+
+# chown user:group filename
+
+##### List current mount detail
+
+
+df
+
+
+##### List current usernames and user-numbers
+
+
+cat /etc/passwd
+
+##### Get all username
+
+
+getent passwd| awk '{FS="[:]"; print $1}'
+
+
+##### Show all users
+
+
+compgen -u
+
+
+##### Show all groups
+
+
+compgen -g
+
+
+##### Show group of user
+
+
+group username
+
+
+##### Show uid, gid, group of user
+
+
+id username
+
+
+##### Check if it's root
+
+
+if [$(id -u) -ne 0];then
+    echo "You are not root!"
+    exit;
+fi
+
+# 'id -u' output 0 if it's not root
+
+##### Find out CPU information
+
+
+more /proc/cpuinfo
+
+# or
+
+
+lscpu
+
+
+##### Set quota for user (e.g. disk soft limit: 120586240; hard limit: 125829120)
+
+
+setquota username 120586240 125829120 0 0 /home
+
+
+##### Show quota for user
+
+
+quota -v username
+
+
+##### Fork bomb
+
+
+# :(){:|:&};:
+
+# dont try this at home
+
+##### Check user login
+
+
+lastlog
+
+
+##### Edit path for all users
+
+
+joe /etc/environment
+
+# edit this file
+
+##### Show running processes
+
+
+ps aux
+
+
+##### Find maximum number of processes
+
+
+cat /proc/sys/kernal/pid_max
+
+
+##### Show and set user limit
+
+
+ulimit -u
+
+##### Which ports are listening for TCP connections from the network
+
+
+nmap -sT -O localhost
+
+
+##### Print out number of cores/ processors
+
+
+nproc --all
+
+##### Check status of each core
+# 1. top  
+# 2. press '1'
+
+##### Show jobs and PID
+
+
+jobs -l
+
+
+##### List all running services
+
+
+service --status-all
+
+
+##### Schedule shutdown server
+
+shutdown -r +5 "Server will restart in 5 minutes. Please save your work."
+
+##### Cancel scheduled shutdown
+
+shutdown -c
+
+
+##### Boardcast to all users
+
+wall -n hihi
+
+##### Kill all process of a user
+
+pkill -U user_name
+
+##### Set gedit preference on server
+
+# -->you might have to install the following:
+
+apt-get install libglib2.0-bin;  
+
+yum install dconf dconf-editor;  
+yum install dbus dbus-x11;  
+
+# -->Check list  
+
+gsettings list-recursively
+
+# -->Change setting  
+# e.g.
+
+gsettings set org.gnome.gedit.preferences.editor highlight-current-line true
+gsettings set org.gnome.gedit.preferences.editor scheme 'cobalt'
+gsettings set org.gnome.gedit.preferences.editor use-default-font false
+gsettings set org.gnome.gedit.preferences.editor editor-font 'Cantarell Regular 12'
+
+##### Find out who has logged in on your system
+--> [Quick] Printing out only the names:
+
+users
+
+
+# --> [Detail] Printing out login time, load average, etc
+
+w
+
+##### Add user to a group (e.g add user 'nice' to the group 'docker', so that he can run docker without sudo) 
+
+sudo gpasswd -a nice docker
+
+
+##### pip install python package without root 
+
+# 1. pip install --user package_name
+# 2. You might need to export ~/.local/bin/ to PATH: export PATH=$PATH:~/.local/bin/
+
+
+##### Removing old linux kernels (when /boot almost full...)
+
+# 1. uname -a  #check current kernel, which should NOT be removed
+# 2. sudo apt-get purge linux-image-X.X.X-X-generic  #replace old version
+
+
+
+##### Change hostname
+
+sudo hostname your-new-name
+
+# if not working, do also:  
+
+hostnamectl set-hostname your-new-hostname
+
+then run:  
+hostnamectl
+
+# check /etc/hostname  
+
+# if still not working..., edit:  
+# /etc/sysconfig/network  
+# /etc/sysconfig/network-scripts/ifcfg-ensxxx  
+# add HOSTNAME="your-new-hostname"  
+
+
+##### List installed packages
+
+apt list --installed
+
+# or Red Hat: 
+
+yum list installed
+
+##### Tutorial for setting up your own DNS server
+# http://onceuponmine.blogspot.tw/2017/08/set-up-your-own-dns-server.html
+
+##### Tutorial for creating a simple daemon
+# http://onceuponmine.blogspot.tw/2017/07/create-your-first-simple-daemon.html
+
+
+
+# =-=-=-=-=-A lot more coming!! =-=-=-=-=-=-=-=-=-=waitwait-=-=-=-=-=-=-=-=-=-
No results found