|
|
@@ -1,461 +1,471 @@ |
|
|
General Usage |
|
|
------------- |
|
|
|
|
|
Most of the following commands can be used both on files directly and in pipes, e.g. the command |
|
|
`sed G` can be used in both of the following ways: |
|
|
|
|
|
... | sed G |
|
|
sed G [/path/to/]file |
|
|
|
|
|
|
|
|
File Spacing |
|
|
------------ |
|
|
|
|
|
Double space: |
|
|
|
|
|
$ sed G |
|
|
$ awk '1;{print ""}' |
|
|
$ awk 'BEGIN{ORS="\n\n"};1' |
|
|
sed G |
|
|
awk '1;{print ""}' |
|
|
awk 'BEGIN{ORS="\n\n"};1' |
|
|
|
|
|
Double space which already has blank lines in it. Output should contain no more than one blank line |
|
|
between lines of text. |
|
|
|
|
|
$ sed '/^$/d;G' |
|
|
$ awk 'NF{print $0 "\n"}' |
|
|
sed '/^$/d;G' |
|
|
awk 'NF{print $0 "\n"}' |
|
|
|
|
|
Triple space: |
|
|
|
|
|
$ sed 'G;G' |
|
|
$ awk '1;{print "\n"}' |
|
|
sed 'G;G' |
|
|
awk '1;{print "\n"}' |
|
|
|
|
|
Undo double-spacing (assumes even-numbered lines are always blank): |
|
|
|
|
|
$ sed 'n;d' |
|
|
sed 'n;d' |
|
|
|
|
|
Insert a blank line above every line which matches `regex`: |
|
|
|
|
|
$ sed '/regex/{x;p;x;}' |
|
|
sed '/regex/{x;p;x;}' |
|
|
|
|
|
Insert a blank line below every line which matches `regex`: |
|
|
|
|
|
$ sed '/regex/G' |
|
|
sed '/regex/G' |
|
|
|
|
|
Insert a blank line above and below every line which matches the regular expression: |
|
|
|
|
|
$ sed '/regex/{x;p;x;G;}' |
|
|
sed '/regex/{x;p;x;G;}' |
|
|
|
|
|
|
|
|
Numbering and Calculations |
|
|
-------------------------- |
|
|
|
|
|
Number each line (simple left alignment), using a tab: |
|
|
|
|
|
$ sed = filename | sed 'N;s/\n/\t/' |
|
|
$ awk '{print FNR "\t" $0}' |
|
|
sed = filename | sed 'N;s/\n/\t/' |
|
|
awk '{print FNR "\t" $0}' |
|
|
|
|
|
Precede each line by its line number FOR ALL FILES TOGETHER, using a tab: |
|
|
|
|
|
$ awk '{print NR "\t" $0}' files* |
|
|
awk '{print NR "\t" $0}' files* |
|
|
|
|
|
Number each line (number on left, right-aligned): |
|
|
|
|
|
$ sed = filename | sed 'N; s/^/ /; s/ *\(.\{6,\}\)\n/\1 /' |
|
|
$ awk '{printf("%5d : %s\n", NR,$0)}' |
|
|
sed = filename | sed 'N; s/^/ /; s/ *\(.\{6,\}\)\n/\1 /' |
|
|
awk '{printf("%5d : %s\n", NR,$0)}' |
|
|
|
|
|
Number each line, but only print numbers if line is not blank: |
|
|
|
|
|
$ sed '/./=' filename | sed '/./N; s/\n/ /' |
|
|
$ awk 'NF{$0=++a " :" $0};1' |
|
|
$ awk '{print (NF? ++a " :" :"") $0}' |
|
|
sed '/./=' filename | sed '/./N; s/\n/ /' |
|
|
awk 'NF{$0=++a " :" $0};1' |
|
|
awk '{print (NF? ++a " :" :"") $0}' |
|
|
|
|
|
Count lines: |
|
|
|
|
|
$ wc -l |
|
|
$ sed -n '$=' |
|
|
$ awk 'END{print NR}' |
|
|
wc -l |
|
|
sed -n '$=' |
|
|
awk 'END{print NR}' |
|
|
|
|
|
Print the sums of the fields of every line: |
|
|
|
|
|
$ awk '{s=0; for (i=1; i<=NF; i++) s=s+$i; print s}' |
|
|
awk '{s=0; for (i=1; i<=NF; i++) s=s+$i; print s}' |
|
|
|
|
|
Add all fields in all lines and print the sum: |
|
|
|
|
|
$ awk '{for (i=1; i<=NF; i++) s=s+$i}; END{print s}' |
|
|
awk '{for (i=1; i<=NF; i++) s=s+$i}; END{print s}' |
|
|
|
|
|
Print every line after replacing each field with its absolute value: |
|
|
|
|
|
$ awk '{for (i=1; i<=NF; i++) if ($i < 0) $i = -$i; print }' |
|
|
$ awk '{for (i=1; i<=NF; i++) $i = ($i < 0) ? -$i : $i; print }' |
|
|
awk '{for (i=1; i<=NF; i++) if ($i < 0) $i = -$i; print }' |
|
|
awk '{for (i=1; i<=NF; i++) $i = ($i < 0) ? -$i : $i; print }' |
|
|
|
|
|
Print the total number of fields ("words") in all lines: |
|
|
|
|
|
$ awk '{ total = total + NF }; END {print total}' file |
|
|
awk '{ total = total + NF }; END {print total}' file |
|
|
|
|
|
Print the total number of lines that contain "Beth": |
|
|
|
|
|
$ awk '/Beth/{n++}; END {print n+0}' |
|
|
awk '/Beth/{n++}; END {print n+0}' |
|
|
|
|
|
Print the largest first field and the line that contains it (intended for finding the longest string |
|
|
in field 1): |
|
|
|
|
|
$ awk '$1 > max {max=$1; maxline=$0}; END{ print max, maxline}' |
|
|
awk '$1 > max {max=$1; maxline=$0}; END{ print max, maxline}' |
|
|
|
|
|
Print the number of fields in each line, followed by the line: |
|
|
|
|
|
$ awk '{ print NF ":" $0 } ' |
|
|
awk '{ print NF ":" $0 } ' |
|
|
|
|
|
Print the last field of each line: |
|
|
|
|
|
$ awk '{ print $NF }' |
|
|
awk '{ print $NF }' |
|
|
|
|
|
Print the last field of the last line: |
|
|
|
|
|
$ awk '{ field = $NF }; END{ print field }' |
|
|
awk '{ field = $NF }; END{ print field }' |
|
|
|
|
|
Print every line with more than 4 fields: |
|
|
|
|
|
$ awk 'NF > 4' |
|
|
awk 'NF > 4' |
|
|
|
|
|
Print every line where the value of the last field is > 4 |
|
|
|
|
|
$ awk '$NF > 4' |
|
|
awk '$NF > 4' |
|
|
|
|
|
Add first and last column in each line: |
|
|
|
|
|
$ perl -lane 'print $F[0] + $F[-1]' |
|
|
perl -lane 'print $F[0] + $F[-1]' |
|
|
|
|
|
Increments all numbers by 1: |
|
|
|
|
|
$ perl -pe 's/(\d+)/ 1 + $1 /ge' |
|
|
perl -pe 's/(\d+)/ 1 + $1 /ge' |
|
|
|
|
|
Sum up a column: |
|
|
|
|
|
$ awk '{ sum+=$1 } END { print sum }' |
|
|
awk '{ sum+=$1 } END { print sum }' |
|
|
|
|
|
|
|
|
Text Conversion and Substitution |
|
|
-------------------------------- |
|
|
|
|
|
Convert newline to space: |
|
|
|
|
|
$ tr '\n' ' ' |
|
|
$ perl -pe 's/\n/ /' |
|
|
$ sed ':a;N;$!ba;s/\n/ /g' # will not convert last newline! |
|
|
tr '\n' ' ' |
|
|
perl -pe 's/\n/ /' |
|
|
sed ':a;N;$!ba;s/\n/ /g' # will not convert last newline! |
|
|
|
|
|
Convert CRLF to LF format: |
|
|
|
|
|
$ sed 's/.$//' # assumes that all lines end with CRLF |
|
|
$ sed 's/^M$//' # in bash/tcsh, press Ctrl-V then Ctrl-M |
|
|
$ sed 's/\x0D$//' # works on ssed, gsed 3.02.80 or higher |
|
|
$ tr -d \r # GNU tr version 1.22 or higher |
|
|
$ awk '{sub(/\r$/,"")};1' # assumes EACH line ends with Ctrl-M |
|
|
$ perl -p -i -e 's/\012?\015/\n/g' |
|
|
sed 's/.$//' # assumes that all lines end with CRLF |
|
|
sed 's/^M$//' # in bash/tcsh, press Ctrl-V then Ctrl-M |
|
|
sed 's/\x0D$//' # works on ssed, gsed 3.02.80 or higher |
|
|
tr -d \r # GNU tr version 1.22 or higher |
|
|
awk '{sub(/\r$/,"")};1' # assumes EACH line ends with Ctrl-M |
|
|
perl -p -i -e 's/\012?\015/\n/g' |
|
|
|
|
|
|
|
|
Delete leading whitespace (spaces, tabs) from front of each line, aligns all text, flush left: |
|
|
|
|
|
$ sed 's/^[ \t]*//' |
|
|
$ awk '{sub(/^[ \t]+/, "")};1' |
|
|
sed 's/^[ \t]*//' |
|
|
awk '{sub(/^[ \t]+/, "")};1' |
|
|
|
|
|
Delete trailing whitespace (spaces, tabs) from end of each line: |
|
|
|
|
|
$ sed 's/[ \t]*$//' |
|
|
$ awk '{sub(/[ \t]+$/, "")};1' |
|
|
sed 's/[ \t]*$//' |
|
|
awk '{sub(/[ \t]+$/, "")};1' |
|
|
|
|
|
Delete both leading and trailing whitespace from each line: |
|
|
|
|
|
$ sed 's/^[ \t]*//;s/[ \t]*$//' |
|
|
$ awk '{gsub(/^[ \t]+|[ \t]+$/,"")};1' |
|
|
$ awk '{$1=$1};1' # also removes extra space between fields |
|
|
sed 's/^[ \t]*//;s/[ \t]*$//' |
|
|
awk '{gsub(/^[ \t]+|[ \t]+$/,"")};1' |
|
|
awk '{$1=$1};1' # also removes extra space between fields |
|
|
|
|
|
Insert 5 blank spaces at beginning of each line (make page offset): |
|
|
|
|
|
$ sed 's/^/ /' |
|
|
$ awk '{sub(/^/, " ")};1' |
|
|
sed 's/^/ /' |
|
|
awk '{sub(/^/, " ")};1' |
|
|
|
|
|
Align all text, flush right on a 79-column width: |
|
|
|
|
|
$ sed -e :a -e 's/^.\{1,78\}$/ &/;ta' |
|
|
$ awk '{printf "%79s\n", $0}' |
|
|
sed -e :a -e 's/^.\{1,78\}$/ &/;ta' |
|
|
awk '{printf "%79s\n", $0}' |
|
|
|
|
|
Center all text in the middle of 79-column width. In method 1, spaces at the beginning of the line |
|
|
are significant, and trailing spaces are appended at the end of the line. In method 2, spaces at the |
|
|
beginning of the line are discarded in centering the line, and no trailing spaces appear at the end |
|
|
of lines. |
|
|
|
|
|
$ sed -e :a -e 's/^.\{1,77\}$/ & /;ta' # method 1 |
|
|
$ sed -e :a -e 's/^.\{1,77\}$/ &/;ta' -e 's/\( *\)\1/\1/' # method 2 |
|
|
$ awk '{l=length();s=int((79-l)/2); printf "%"(s+l)"s\n",$0}' |
|
|
sed -e :a -e 's/^.\{1,77\}$/ & /;ta' # method 1 |
|
|
sed -e :a -e 's/^.\{1,77\}$/ &/;ta' -e 's/\( *\)\1/\1/' # method 2 |
|
|
awk '{l=length();s=int((79-l)/2); printf "%"(s+l)"s\n",$0}' |
|
|
|
|
|
Substitute (find and replace) `foo` with `bar` on each line: |
|
|
|
|
|
$ sed 's/foo/bar/' # replaces only 1st instance in a line |
|
|
$ awk '{sub(/foo/,"bar")}; 1' # replaces only 1st instance in a line |
|
|
$ sed 's/foo/bar/4' # replaces only 4th instance in a line |
|
|
$ gawk '{$0=gensub(/foo/,"bar",4)}; 1' # replaces only 4th instance in a line |
|
|
$ sed 's/foo/bar/g' # replaces ALL instances in a line |
|
|
$ awk '{gsub(/foo/,"bar")}; 1' # replaces ALL instances in a line |
|
|
$ sed 's/\(.*\)foo\(.*foo\)/\1bar\2/' # replaces the next-to-last case |
|
|
$ sed 's/\(.*\)foo/\1bar/' # replaces only the last case |
|
|
$ perl -p -i.bak -e 's/foo/bar/g' *.c # also keeps backups |
|
|
sed 's/foo/bar/' # replaces only 1st instance in a line |
|
|
awk '{sub(/foo/,"bar")}; 1' # replaces only 1st instance in a line |
|
|
sed 's/foo/bar/4' # replaces only 4th instance in a line |
|
|
gawk '{$0=gensub(/foo/,"bar",4)}; 1' # replaces only 4th instance in a line |
|
|
sed 's/foo/bar/g' # replaces ALL instances in a line |
|
|
awk '{gsub(/foo/,"bar")}; 1' # replaces ALL instances in a line |
|
|
sed 's/\(.*\)foo\(.*foo\)/\1bar\2/' # replaces the next-to-last case |
|
|
sed 's/\(.*\)foo/\1bar/' # replaces only the last case |
|
|
perl -p -i.bak -e 's/foo/bar/g' *.c # also keeps backups |
|
|
|
|
|
Substitute `foo` with `bar` ONLY for lines which contain `baz`: |
|
|
|
|
|
$ sed '/baz/s/foo/bar/g' |
|
|
$ awk '/baz/{gsub(/foo/, "bar")}; 1' |
|
|
sed '/baz/s/foo/bar/g' |
|
|
awk '/baz/{gsub(/foo/, "bar")}; 1' |
|
|
|
|
|
Substitute `foo` with `bar` except for lines which contain `baz`: |
|
|
|
|
|
$ sed '/baz/!s/foo/bar/g' |
|
|
$ awk '!/baz/{gsub(/foo/, "bar")}; 1' |
|
|
sed '/baz/!s/foo/bar/g' |
|
|
awk '!/baz/{gsub(/foo/, "bar")}; 1' |
|
|
|
|
|
Change `scarlet` or `ruby` or `puce` to `red`: |
|
|
|
|
|
$ sed 's/scarlet/red/g;s/ruby/red/g;s/puce/red/g' # most seds |
|
|
$ gsed 's/scarlet\|ruby\|puce/red/g' # GNU sed only |
|
|
$ awk '{gsub(/scarlet|ruby|puce/, "red")}; 1' |
|
|
sed 's/scarlet/red/g;s/ruby/red/g;s/puce/red/g' # most seds |
|
|
gsed 's/scarlet\|ruby\|puce/red/g' # GNU sed only |
|
|
awk '{gsub(/scarlet|ruby|puce/, "red")}; 1' |
|
|
|
|
|
Reverse order of lines: |
|
|
|
|
|
$ tac |
|
|
$ sed '1!G;h;$!d' |
|
|
$ sed -n '1!G;h;$p' |
|
|
$ awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }' |
|
|
$ perl -e 'print reverse <>' |
|
|
tac |
|
|
sed '1!G;h;$!d' |
|
|
sed -n '1!G;h;$p' |
|
|
awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }' |
|
|
perl -e 'print reverse <>' |
|
|
|
|
|
Reverses paragraphs: |
|
|
|
|
|
$ perl -00 -e 'print reverse <>' |
|
|
perl -00 -e 'print reverse <>' |
|
|
|
|
|
Reverse each character on the line: |
|
|
|
|
|
$ rev |
|
|
$ sed '/\n/!G;s/\(.\)\(.*\n\)/&\2\1/;//D;s/.//' |
|
|
$ perl -nle 'print scalar reverse $_' |
|
|
rev |
|
|
sed '/\n/!G;s/\(.\)\(.*\n\)/&\2\1/;//D;s/.//' |
|
|
perl -nle 'print scalar reverse $_' |
|
|
|
|
|
Join pairs of lines side-by-side: |
|
|
|
|
|
$ paste |
|
|
$ sed '$!N;s/\n/ /' |
|
|
paste |
|
|
sed '$!N;s/\n/ /' |
|
|
|
|
|
If a line ends with a backslash, append the next line to it: |
|
|
|
|
|
$ sed -e :a -e '/\\$/N; s/\\\n//; ta' |
|
|
sed -e :a -e '/\\$/N; s/\\\n//; ta' |
|
|
|
|
|
If a line begins with an equal sign, append it to the previous line and replace the "=" with a |
|
|
single space: |
|
|
|
|
|
$ sed -e :a -e '$!N;s/\n=/ /;ta' -e 'P;D' |
|
|
sed -e :a -e '$!N;s/\n=/ /;ta' -e 'P;D' |
|
|
|
|
|
Add commas to numeric strings, changing "1234567" to "1,234,567": |
|
|
|
|
|
$ gsed ':a;s/\B[0-9]\{3\}\>/,&/;ta' # GNU sed |
|
|
$ sed -e :a -e 's/\(.*[0-9]\)\([0-9]\{3\}\)/\1,\2/;ta' # other seds |
|
|
gsed ':a;s/\B[0-9]\{3\}\>/,&/;ta' # GNU sed |
|
|
sed -e :a -e 's/\(.*[0-9]\)\([0-9]\{3\}\)/\1,\2/;ta' # other seds |
|
|
|
|
|
Add commas to numbers with decimal points and minus signs: |
|
|
|
|
|
$ gsed -r ':a;s/(^|[^0-9.])([0-9]+)([0-9]{3})/\1\2,\3/g;ta' |
|
|
gsed -r ':a;s/(^|[^0-9.])([0-9]+)([0-9]{3})/\1\2,\3/g;ta' |
|
|
|
|
|
Add a blank line every 5 lines (after lines 5, 10, 15, 20, etc.): |
|
|
|
|
|
$ gsed '0~5G' # GNU sed only |
|
|
$ sed 'n;n;n;n;G;' # other seds |
|
|
gsed '0~5G' # GNU sed only |
|
|
sed 'n;n;n;n;G;' # other seds |
|
|
|
|
|
Print and sort the login names of all users: |
|
|
|
|
|
$ awk -F ":" '{print $1 | "sort" }' /etc/passwd |
|
|
awk -F ":" '{print $1 | "sort" }' /etc/passwd |
|
|
|
|
|
Print the first 2 fields, in opposite order, of every line: |
|
|
|
|
|
$ awk '{print $2, $1}' |
|
|
awk '{print $2, $1}' |
|
|
|
|
|
Switch the first 2 fields of every line: |
|
|
|
|
|
$ awk '{temp = $1; $1 = $2; $2 = temp}' |
|
|
awk '{temp = $1; $1 = $2; $2 = temp}' |
|
|
|
|
|
Print every line, deleting the first field of that line: |
|
|
|
|
|
$ cut -d' ' -f2- |
|
|
$ awk '{ $1 = ""; print }' |
|
|
cut -d' ' -f2- |
|
|
awk '{ $1 = ""; print }' |
|
|
|
|
|
Print every line, deleting the second field of that line: |
|
|
|
|
|
$ awk '{ $2 = ""; print }' |
|
|
awk '{ $2 = ""; print }' |
|
|
|
|
|
Print in reverse order the fields of every line: |
|
|
|
|
|
$ awk '{for (i=NF; i>0; i--) printf("%s ",$i);print ""}' |
|
|
awk '{for (i=NF; i>0; i--) printf("%s ",$i);print ""}' |
|
|
|
|
|
Concatenate every 5 lines of input, using a comma separator between fields: |
|
|
|
|
|
$ awk 'ORS=NR%5?",":"\n"' |
|
|
awk 'ORS=NR%5?",":"\n"' |
|
|
|
|
|
Rename in each file name the string `aaa` by `bbb`: |
|
|
|
|
|
$ ls | perl -ne 'chomp; next unless -e; $o = $_; s/aaa/bbb/; next if -e; rename $o, $_'; |
|
|
ls | perl -ne 'chomp; next unless -e; $o = $_; s/aaa/bbb/; next if -e; rename $o, $_'; |
|
|
|
|
|
|
|
|
Selective Printing of Certain Lines |
|
|
----------------------------------- |
|
|
|
|
|
Print first 10 lines: |
|
|
|
|
|
$ head |
|
|
$ sed 10q |
|
|
$ awk 'NR < 11' |
|
|
head |
|
|
sed 10q |
|
|
awk 'NR < 11' |
|
|
|
|
|
Print first line: |
|
|
|
|
|
$ head -1 |
|
|
$ sed q |
|
|
$ awk 'NR>1{exit};1' |
|
|
head -1 |
|
|
sed q |
|
|
awk 'NR>1{exit};1' |
|
|
|
|
|
Print last 10 lines: |
|
|
|
|
|
$ tail |
|
|
$ sed -e :a -e '$q;N;11,$D;ba' |
|
|
tail |
|
|
sed -e :a -e '$q;N;11,$D;ba' |
|
|
|
|
|
Print last 2 lines: |
|
|
|
|
|
$ tail -2 |
|
|
$ sed '$!N;$!D' |
|
|
$ awk '{y=x "\n" $0; x=$0};END{print y}' |
|
|
tail -2 |
|
|
sed '$!N;$!D' |
|
|
awk '{y=x "\n" $0; x=$0};END{print y}' |
|
|
|
|
|
Print last line: |
|
|
|
|
|
$ tail -1 |
|
|
$ sed '$!d' |
|
|
$ sed -n '$p' |
|
|
$ awk 'END{print}' |
|
|
tail -1 |
|
|
sed '$!d' |
|
|
sed -n '$p' |
|
|
awk 'END{print}' |
|
|
|
|
|
Print next-to-the-last line: |
|
|
|
|
|
$ sed -e '$!{h;d;}' -e x # for 1-line, print blank line |
|
|
$ sed -e '1{$q;}' -e '$!{h;d;}' -e x # for 1-line, print the line |
|
|
$ sed -e '1{$d;}' -e '$!{h;d;}' -e x # for 1-line, print nothing |
|
|
sed -e '$!{h;d;}' -e x # for 1-line, print blank line |
|
|
sed -e '1{$q;}' -e '$!{h;d;}' -e x # for 1-line, print the line |
|
|
sed -e '1{$d;}' -e '$!{h;d;}' -e x # for 1-line, print nothing |
|
|
|
|
|
Print only lines which match regular expression: |
|
|
|
|
|
$ grep 'regex' |
|
|
$ sed -n '/regex/p' # method 1 |
|
|
$ sed '/regex/!d' # method 2 |
|
|
$ awk '/regex/' |
|
|
grep 'regex' |
|
|
sed -n '/regex/p' # method 1 |
|
|
sed '/regex/!d' # method 2 |
|
|
awk '/regex/' |
|
|
|
|
|
Print only lines which do not match regular expression: |
|
|
|
|
|
$ grep -v regex |
|
|
$ sed -n '/regex/!p' # method 1, corresponds to above |
|
|
$ sed '/regex/d' # method 2, simpler syntax |
|
|
$ awk '!/regex/' |
|
|
grep -v regex |
|
|
sed -n '/regex/!p' # method 1, corresponds to above |
|
|
sed '/regex/d' # method 2, simpler syntax |
|
|
awk '!/regex/' |
|
|
|
|
|
Print the line immediately before a regular expression, but not the line containing it: |
|
|
|
|
|
$ sed -n '/regex/{g;1!p;};h' |
|
|
$ awk '/regex/{print x};{x=$0}' |
|
|
$ awk '/regex/{print (NR==1 ? "match on line 1" : x)};{x=$0}' |
|
|
sed -n '/regex/{g;1!p;};h' |
|
|
awk '/regex/{print x};{x=$0}' |
|
|
awk '/regex/{print (NR==1 ? "match on line 1" : x)};{x=$0}' |
|
|
|
|
|
Print the line immediately after a regular expression, but not the line containing it: |
|
|
|
|
|
$ sed -n '/regex/{n;p;}' |
|
|
$ awk '/regex/{getline;print}' |
|
|
sed -n '/regex/{n;p;}' |
|
|
awk '/regex/{getline;print}' |
|
|
|
|
|
Print 1 line of context before and after a regular expression, with line numbers: |
|
|
|
|
|
$ grep -A1 -B1 -n regex |
|
|
$ sed -n -e '/regex/{=;x;1!p;g;$!N;p;D;}' -e h |
|
|
grep -A1 -B1 -n regex |
|
|
sed -n -e '/regex/{=;x;1!p;g;$!N;p;D;}' -e h |
|
|
|
|
|
Search for `AAA` and `BBB` and `CCC` (in any order): |
|
|
|
|
|
$ sed '/AAA/!d; /BBB/!d; /CCC/!d' |
|
|
$ awk '/AAA/ && /BBB/ && /CCC/' |
|
|
sed '/AAA/!d; /BBB/!d; /CCC/!d' |
|
|
awk '/AAA/ && /BBB/ && /CCC/' |
|
|
|
|
|
Search for `AAA` and `BBB` and `CCC` (in that order): |
|
|
|
|
|
$ sed '/AAA.*BBB.*CCC/!d' |
|
|
$ awk '/AAA.*BBB.*CCC/' |
|
|
sed '/AAA.*BBB.*CCC/!d' |
|
|
awk '/AAA.*BBB.*CCC/' |
|
|
|
|
|
Search for `AAA` or `BBB` or `CCC`: |
|
|
|
|
|
$ egrep "AAA|BBB|CCC" |
|
|
$ grep -E "AAA|BBB|CCC" |
|
|
$ sed -e '/AAA/b' -e '/BBB/b' -e '/CCC/b' -e d # most seds |
|
|
$ gsed '/AAA\|BBB\|CCC/!d' # GNU sed only |
|
|
egrep "AAA|BBB|CCC" |
|
|
grep -E "AAA|BBB|CCC" |
|
|
sed -e '/AAA/b' -e '/BBB/b' -e '/CCC/b' -e d # most seds |
|
|
gsed '/AAA\|BBB\|CCC/!d' # GNU sed only |
|
|
|
|
|
Print paragraph if it contains `AAA` (blank lines separate paragraphs): |
|
|
|
|
|
$ sed -e '/./{H;$!d;}' -e 'x;/AAA/!d;' |
|
|
sed -e '/./{H;$!d;}' -e 'x;/AAA/!d;' |
|
|
|
|
|
Print paragraph if it contains `AAA` and `BBB` and `CCC` (in any order): |
|
|
|
|
|
$ sed -e '/./{H;$!d;}' -e 'x;/AAA/!d;/BBB/!d;/CCC/!d' |
|
|
sed -e '/./{H;$!d;}' -e 'x;/AAA/!d;/BBB/!d;/CCC/!d' |
|
|
|
|
|
Print paragraph if it contains `AAA` or `BBB` or `CCC`: |
|
|
|
|
|
$ sed -e '/./{H;$!d;}' -e 'x;/AAA/b' -e '/BBB/b' -e '/CCC/b' -e d |
|
|
$ gsed '/./{H;$!d;};x;/AAA\|BBB\|CCC/b;d' # GNU sed only |
|
|
sed -e '/./{H;$!d;}' -e 'x;/AAA/b' -e '/BBB/b' -e '/CCC/b' -e d |
|
|
gsed '/./{H;$!d;};x;/AAA\|BBB\|CCC/b;d' # GNU sed only |
|
|
|
|
|
Print only lines of 65 characters or longer: |
|
|
|
|
|
$ sed -n '/^.\{65\}/p' |
|
|
$ awk 'length > 64' |
|
|
sed -n '/^.\{65\}/p' |
|
|
awk 'length > 64' |
|
|
|
|
|
Print only lines of less than 65 characters: |
|
|
|
|
|
$ sed -n '/^.\{65\}/!p' # method 1, corresponds to above |
|
|
$ sed '/^.\{65\}/d' # method 2, simpler syntax |
|
|
$ awk 'length < 65' |
|
|
sed -n '/^.\{65\}/!p' # method 1, corresponds to above |
|
|
sed '/^.\{65\}/d' # method 2, simpler syntax |
|
|
awk 'length < 65' |
|
|
|
|
|
Print section from regular expression to end: |
|
|
|
|
|
$ sed -n '/regex/,$p' |
|
|
$ awk '/regex/,0' |
|
|
$ awk '/regex/,EOF' |
|
|
sed -n '/regex/,$p' |
|
|
awk '/regex/,0' |
|
|
awk '/regex/,EOF' |
|
|
|
|
|
Print section based on line numbers (lines 8-12, inclusive): |
|
|
|
|
|
$ sed -n '8,12p' |
|
|
$ sed '8,12!d' |
|
|
$ awk 'NR==8,NR==12' |
|
|
$ perl -ne 'print if 8 .. 12' |
|
|
$ perl -pe 'exit if 8<$. && $.<12' |
|
|
sed -n '8,12p' |
|
|
sed '8,12!d' |
|
|
awk 'NR==8,NR==12' |
|
|
perl -ne 'print if 8 .. 12' |
|
|
perl -pe 'exit if 8<$. && $.<12' |
|
|
|
|
|
Print line number 52: |
|
|
|
|
|
$ sed -n '52p' |
|
|
$ sed '52!d' |
|
|
$ sed '52q;d' # efficient on large files |
|
|
$ awk 'NR==52' |
|
|
$ awk 'NR==52 {print;exit}' # more efficient on large files |
|
|
sed -n '52p' |
|
|
sed '52!d' |
|
|
sed '52q;d' # efficient on large files |
|
|
awk 'NR==52' |
|
|
awk 'NR==52 {print;exit}' # more efficient on large files |
|
|
|
|
|
Beginning at line 3, print every 7th line: |
|
|
|
|
|
$ gsed -n '3~7p' |
|
|
$ sed -n '3,${p;n;n;n;n;n;n;}' |
|
|
gsed -n '3~7p' |
|
|
sed -n '3,${p;n;n;n;n;n;n;}' |
|
|
|
|
|
Print section between two regular expressions (inclusive): |
|
|
|
|
|
$ sed -n '/START/,/END/p' |
|
|
$ awk '/START/,/END/' |
|
|
$ perl -ne 'print if /START/ .. /END/' |
|
|
$ perl -ne 'print if m{START} .. m{END}' |
|
|
sed -n '/START/,/END/p' |
|
|
awk '/START/,/END/' |
|
|
perl -ne 'print if /START/ .. /END/' |
|
|
perl -ne 'print if m{START} .. m{END}' |
|
|
|
|
|
Print all except section between 2 regular expressions: |
|
|
|
|
|
$ sed '/START/,/END/d' |
|
|
$ perl -i.old -ne 'print unless /START/ .. /END/' |
|
|
sed '/START/,/END/d' |
|
|
perl -i.old -ne 'print unless /START/ .. /END/' |
|
|
|
|
|
Match a field against a regular expression: |
|
|
|
|
|
$ awk '$7 ~ /^[a-f]/' # print line if field #7 matches regex |
|
|
$ awk '$7 !~ /^[a-f]/' # print line if field #7 does NOT match regex |
|
|
awk '$7 ~ /^[a-f]/' # print line if field #7 matches regex |
|
|
awk '$7 !~ /^[a-f]/' # print line if field #7 does NOT match regex |
|
|
|
|
|
Print any line where field 5 is equal to `abc123`: |
|
|
|
|
|
$ awk '$5 == "abc123"' |
|
|
awk '$5 == "abc123"' |
|
|
|
|
|
Print only those lines where field 5 is NOT equal to `abc123` (also prints lines with less than 5 |
|
|
fields): |
|
|
|
|
|
$ awk '$5 != "abc123"' |
|
|
$ awk '!($5 == "abc123")' |
|
|
awk '$5 != "abc123"' |
|
|
awk '!($5 == "abc123")' |
|
|
|
|
|
Print palindromes: |
|
|
|
|
|
$ perl -lne 'print if $_ eq reverse' |
|
|
perl -lne 'print if $_ eq reverse' |
|
|
|
|
|
Print duplicated words in a line: |
|
|
|
|
|
$ perl -0777 -ne 'print "$.: doubled $_\n" while /\b(\w+)\b\s+\b\1\b/gi' |
|
|
perl -0777 -ne 'print "$.: doubled $_\n" while /\b(\w+)\b\s+\b\1\b/gi' |
|
|
|
|
|
|
|
|
Selective Deletion of Certain Lines |
|
|
@@ -464,92 +474,92 @@ Selective Deletion of Certain Lines |
|
|
Delete duplicate, consecutive lines, first line in a set of duplicate lines is kept, rest are |
|
|
deleted: |
|
|
|
|
|
$ uniq |
|
|
$ sed '$!N; /^\(.*\)\n\1$/!P; D' |
|
|
$ awk 'a !~ $0; {a=$0}' |
|
|
uniq |
|
|
sed '$!N; /^\(.*\)\n\1$/!P; D' |
|
|
awk 'a !~ $0; {a=$0}' |
|
|
|
|
|
Delete duplicate, nonconsecutive lines: |
|
|
|
|
|
$ sed -n 'G; s/\n/&&/; /^\([ -~]*\n\).*\n\1/d; s/\n//; h; P' |
|
|
$ awk '!a[$0]++' # most concise script |
|
|
$ awk '!($0 in a){a[$0];print}' # most efficient script |
|
|
sed -n 'G; s/\n/&&/; /^\([ -~]*\n\).*\n\1/d; s/\n//; h; P' |
|
|
awk '!a[$0]++' # most concise script |
|
|
awk '!($0 in a){a[$0];print}' # most efficient script |
|
|
|
|
|
Delete all lines except duplicate lines: |
|
|
|
|
|
$ uniq -d |
|
|
$ sed '$!N; s/^\(.*\)\n\1$/\1/; t; D' |
|
|
uniq -d |
|
|
sed '$!N; s/^\(.*\)\n\1$/\1/; t; D' |
|
|
|
|
|
Delete the first 10 lines: |
|
|
|
|
|
$ sed '1,10d' |
|
|
$ awk 'NR > 10' |
|
|
$ perl -ne 'print unless 1 .. 10' |
|
|
sed '1,10d' |
|
|
awk 'NR > 10' |
|
|
perl -ne 'print unless 1 .. 10' |
|
|
|
|
|
Delete the last line: |
|
|
|
|
|
$ sed '$d' |
|
|
sed '$d' |
|
|
|
|
|
Delete the last 2 lines: |
|
|
|
|
|
$ sed 'N;$!P;$!D;$d' |
|
|
sed 'N;$!P;$!D;$d' |
|
|
|
|
|
Delete the last 10 lines: |
|
|
|
|
|
$ sed -e :a -e '$d;N;2,10ba' -e 'P;D' # method 1 |
|
|
$ sed -n -e :a -e '1,10!{P;N;D;};N;ba' # method 2 |
|
|
sed -e :a -e '$d;N;2,10ba' -e 'P;D' # method 1 |
|
|
sed -n -e :a -e '1,10!{P;N;D;};N;ba' # method 2 |
|
|
|
|
|
Delete every 8th line: |
|
|
|
|
|
$ gsed '0~8d' # GNU sed only |
|
|
$ sed 'n;n;n;n;n;n;n;d;' # other seds |
|
|
gsed '0~8d' # GNU sed only |
|
|
sed 'n;n;n;n;n;n;n;d;' # other seds |
|
|
|
|
|
Delete lines matching pattern: |
|
|
|
|
|
$ sed '/pattern/d' |
|
|
sed '/pattern/d' |
|
|
|
|
|
Delete all blank lines: |
|
|
|
|
|
$ grep '.' |
|
|
$ sed '/^$/d' # method 1 |
|
|
$ sed '/./!d' # method 2 |
|
|
$ awk NF |
|
|
$ awk '/./' |
|
|
grep '.' |
|
|
sed '/^$/d' # method 1 |
|
|
sed '/./!d' # method 2 |
|
|
awk NF |
|
|
awk '/./' |
|
|
|
|
|
Delete all consecutive blank lines except the first, also deletes all blank lines from top and end: |
|
|
|
|
|
$ cat -s |
|
|
$ sed '/./,/^$/!d' # method 1, allows 0 blanks at top, 1 at EOF |
|
|
$ sed '/^$/N;/\n$/D' # method 2, allows 1 blank at top, 0 at EOF |
|
|
cat -s |
|
|
sed '/./,/^$/!d' # method 1, allows 0 blanks at top, 1 at EOF |
|
|
sed '/^$/N;/\n$/D' # method 2, allows 1 blank at top, 0 at EOF |
|
|
|
|
|
Delete all consecutive blank lines except the first 2: |
|
|
|
|
|
$ sed '/^$/N;/\n$/N;//D' |
|
|
sed '/^$/N;/\n$/N;//D' |
|
|
|
|
|
Delete all leading blank lines at top: |
|
|
|
|
|
$ sed '/./,$!d' |
|
|
sed '/./,$!d' |
|
|
|
|
|
Delete all trailing blank lines at end: |
|
|
|
|
|
$ sed -e :a -e '/^\n*$/{$d;N;ba' -e '}' # works on all seds |
|
|
$ sed -e :a -e '/^\n*$/N;/\n$/ba' # dito, except for gsed 3.02.* |
|
|
sed -e :a -e '/^\n*$/{$d;N;ba' -e '}' # works on all seds |
|
|
sed -e :a -e '/^\n*$/N;/\n$/ba' # dito, except for gsed 3.02.* |
|
|
|
|
|
Delete the last line of each paragraph: |
|
|
|
|
|
$ sed -n '/^$/{p;h;};/./{x;/./p;}' |
|
|
sed -n '/^$/{p;h;};/./{x;/./p;}' |
|
|
|
|
|
|
|
|
String Creation |
|
|
--------------- |
|
|
|
|
|
Create a string of a specific length (e.g., generate 513 spaces) |
|
|
|
|
|
$ awk 'BEGIN{while (a++<513) s=s " "; print s}' |
|
|
awk 'BEGIN{while (a++<513) s=s " "; print s}' |
|
|
|
|
|
Insert a string of specific length at a certain character position (example inserts 49 spaces after |
|
|
column 6 of each input line): |
|
|
|
|
|
$ gawk --re-interval 'BEGIN{while(a++<49)s=s " "};{sub(/^.{6}/,"&" s)};1' |
|
|
gawk --re-interval 'BEGIN{while(a++<49)s=s " "};{sub(/^.{6}/,"&" s)};1' |
|
|
|
|
|
|
|
|
Special Applications for sed |
|
|
@@ -558,55 +568,55 @@ Special Applications for sed |
|
|
Remove nroff overstrikes (char, backspace) from man pages. The 'echo' command may need an -e switch |
|
|
if you use Unix System V or bash shell. |
|
|
|
|
|
$ sed "s/.`echo \\\b`//g" # double quotes required for Unix environment |
|
|
$ sed 's/.^H//g' # in bash/tcsh, press Ctrl-V and then Ctrl-H |
|
|
$ sed 's/.\x08//g' # hex expression for sed 1.5, GNU sed, ssed |
|
|
sed "s/.`echo \\\b`//g" # double quotes required for Unix environment |
|
|
sed 's/.^H//g' # in bash/tcsh, press Ctrl-V and then Ctrl-H |
|
|
sed 's/.\x08//g' # hex expression for sed 1.5, GNU sed, ssed |
|
|
|
|
|
Get Usenet/e-mail message header: |
|
|
|
|
|
$ sed '/^$/q' # deletes everything after first blank line |
|
|
sed '/^$/q' # deletes everything after first blank line |
|
|
|
|
|
Get Usenet/e-mail message body: |
|
|
|
|
|
$ sed '1,/^$/d' # deletes everything up to first blank line |
|
|
sed '1,/^$/d' # deletes everything up to first blank line |
|
|
|
|
|
Get Subject header, but remove initial "Subject: " portion: |
|
|
|
|
|
$ sed '/^Subject: */!d; s///;q' |
|
|
sed '/^Subject: */!d; s///;q' |
|
|
|
|
|
Get return address header: |
|
|
|
|
|
$ sed '/^Reply-To:/q; /^From:/h; /./d;g;q' |
|
|
sed '/^Reply-To:/q; /^From:/h; /./d;g;q' |
|
|
|
|
|
Parse out the address proper. Pulls out the e-mail address by itself from the 1-line return address |
|
|
header (see preceding script) |
|
|
|
|
|
$ sed 's/ *(.*)//; s/>.*//; s/.*[:<] *//' |
|
|
sed 's/ *(.*)//; s/>.*//; s/.*[:<] *//' |
|
|
|
|
|
Add a leading angle bracket and space to each line (quote a message): |
|
|
|
|
|
$ sed 's/^/> /' |
|
|
sed 's/^/> /' |
|
|
|
|
|
Delete leading angle bracket & space from each line (unquote a message): |
|
|
|
|
|
$ sed 's/^> //' |
|
|
sed 's/^> //' |
|
|
|
|
|
Remove most HTML tags (accommodates multiple-line tags): |
|
|
|
|
|
$ sed -e :a -e 's/<[^>]*>//g;/</N;//ba' |
|
|
sed -e :a -e 's/<[^>]*>//g;/</N;//ba' |
|
|
|
|
|
Extract multi-part uuencoded binaries, removing extraneous header info, so that only the uuencoded |
|
|
portion remains. Files passed to sed must be passed in the proper order. Version 1 can be entered |
|
|
from the command line; version 2 can be made into an executable Unix shell script. |
|
|
|
|
|
$ sed '/^end/,/^begin/d' file1 file2 ... fileX | uudecode # vers. 1 |
|
|
$ sed '/^end/,/^begin/d' "$@" | uudecode # vers. 2 |
|
|
sed '/^end/,/^begin/d' file1 file2 ... fileX | uudecode # vers. 1 |
|
|
sed '/^end/,/^begin/d' "$@" | uudecode # vers. 2 |
|
|
|
|
|
Sort paragraphs of file alphabetically. Paragraphs are separated by blank lines. GNU sed uses \v for |
|
|
vertical tab, or any unique char will do. |
|
|
|
|
|
$ sed '/./{H;d;};x;s/\n/={NL}=/g' file | sort | sed '1s/={NL}=//;s/={NL}=/\n/g' |
|
|
$ gsed '/./{H;d};x;y/\n/\v/' file | sort | sed '1s/\v//;y/\v/\n/' |
|
|
sed '/./{H;d;};x;s/\n/={NL}=/g' file | sort | sed '1s/={NL}=//;s/={NL}=/\n/g' |
|
|
gsed '/./{H;d};x;y/\n/\v/' file | sort | sed '1s/\v//;y/\v/\n/' |
|
|
|
|
|
TYPICAL USE: Sed takes one or more editing commands and applies all of them, in sequence, to each |
|
|
line of input. After all the commands have been applied to the first input line, that line is output |
|
|
@@ -615,9 +625,9 @@ assume that input comes from the standard input device (i.e, the console, normal |
|
|
piped input). One or more filenames can be appended to the command line if the input does not come |
|
|
from stdin. Output is sent to stdout (the screen). Thus: |
|
|
|
|
|
$ cat filename | sed '10q' # uses piped input |
|
|
$ sed '10q' filename # same effect, avoids a useless "cat" |
|
|
$ sed '10q' filename > newfile # redirects output to disk |
|
|
cat filename | sed '10q' # uses piped input |
|
|
sed '10q' filename # same effect, avoids a useless "cat" |
|
|
sed '10q' filename > newfile # redirects output to disk |
|
|
|
|
|
For additional syntax instructions, including the way to apply editing commands from a disk file |
|
|
instead of the command line, consult "sed & awk, 2nd Edition," by Dale Dougherty and Arnold Robbins |
|
|
@@ -649,12 +659,12 @@ editing commands, except at the end of those commands. We have used the syntax w |
|
|
portable to most users of sed, even though the popular GNU versions of sed allow a more succinct |
|
|
syntax. When the reader sees a fairly long command such as this: |
|
|
|
|
|
$ sed -e '/AAA/b' -e '/BBB/b' -e '/CCC/b' -e d |
|
|
sed -e '/AAA/b' -e '/BBB/b' -e '/CCC/b' -e d |
|
|
|
|
|
It is heartening to know that GNU sed will let you reduce it to: |
|
|
|
|
|
$ sed '/AAA/b;/BBB/b;/CCC/b;d' # or even |
|
|
$ sed '/AAA\|BBB\|CCC/b;d' |
|
|
sed '/AAA/b;/BBB/b;/CCC/b;d' # or even |
|
|
sed '/AAA\|BBB\|CCC/b;d' |
|
|
|
|
|
In addition, remember that while many versions of sed accept a command like "/one/ s/RE1/RE2/", some |
|
|
do NOT allow "/one/! s/RE1/RE2/", which contains space before the 's'. Omit the space when typing |
|
|
@@ -664,16 +674,16 @@ OPTIMIZING FOR SPEED: If execution speed needs to be increased (due to large inp |
|
|
processors or hard disks), substitution will be executed more quickly if the "find" expression is |
|
|
specified before giving the "s/.../.../" instruction. Thus: |
|
|
|
|
|
$ sed 's/foo/bar/g' filename # standard replace command |
|
|
$ sed '/foo/ s/foo/bar/g' filename # executes more quickly |
|
|
$ sed '/foo/ s//bar/g' filename # shorthand sed syntax |
|
|
sed 's/foo/bar/g' filename # standard replace command |
|
|
sed '/foo/ s/foo/bar/g' filename # executes more quickly |
|
|
sed '/foo/ s//bar/g' filename # shorthand sed syntax |
|
|
|
|
|
On line selection or deletion in which you only need to output lines from the first part of the |
|
|
file, a "quit" command (q) in the script will drastically reduce processing time for large files. |
|
|
Thus: |
|
|
|
|
|
$ sed -n '45,50p' filename # print line nos. 45-50 of a file |
|
|
$ sed -n '51q;45,50p' filename # same, but executes much faster |
|
|
sed -n '45,50p' filename # print line nos. 45-50 of a file |
|
|
sed -n '51q;45,50p' filename # same, but executes much faster |
|
|
|
|
|
If you have any additional scripts to contribute or if you find errors in this document, please send |
|
|
e-mail to the compiler. Indicate the version of sed you used, the operating system it was compiled |
|
|
@@ -683,4 +693,5 @@ characters or less. Various scripts in this file have been written or contribute |
|
|
Thanks |
|
|
------ |
|
|
|
|
|
Special thanks to the authors of [sed1line](http://sed.sourceforge.net/sed1line.txt) and [awk1line](http://www.pement.org/awk/awk1line.txt) for providing such awesome documents. |
|
|
Special thanks to the authors of [sed1line](http://sed.sourceforge.net/sed1line.txt) and |
|
|
[awk1line](http://www.pement.org/awk/awk1line.txt) for providing such awesome documents. |
wookietreiber revised this gist