more of my good notes

     1  # /usr/bin/zsh
     2
     3  Sun Sep  2 03:25:16 EDT 2018
     4  NOTE:
     5
     6  #!/bin/bash
     7  ####################################
     8  #                      #
     9  # START                #
    10  # g0ne_@p0stal                     #
    11  # ImPerialTeKSolutions@outlook.com #
    12  #                  #
    13  ####################################
    14
    15  #+ ITS_ URL Video DL: batch version 1.0
    16  #* Batch list part of program
    17
    18  # Dimensions
    19  height=200
    20  width=500
    21
    22  # Numerical batch list's
    23  yad --height=$height --width=$width --title "Numerical Answer batch#.list" --entry > /tmp/batch#.$$
    24  _number=`cat /tmp/batch#.$$`
    25
    26  # Batch list
    27  batch_list=batch$_number.list
    28
    29  # Main CMD Variables
    30  cmd_1=$(zenity --width=$width --height=$height --title "Add URL's To Batch List" --text "v1ral_ITS Download, batch verson" --forms --add-entry "URL:" --forms --add-entry "URL:" --add-entry "URL:" --add-entry "URL:" --add-entry "URL:"  --add-entry "URL:"  --add-entry "URL:"  --add-entry "URL:"  --separator='|')
    31
    32  # Start Main Dialog Program
    33  #+ ITS_
    34  echo "$cmd_1" | sed -e "s/|/\n/g" \
    35  | tee -a /home/v1ral/$batch_list
    36
    37  # Finish
    38  /home/v1ral/bin/ITS &&  sign
    39
    40  # Clean and exit
    41  exit $?
    42
    43  _________________________________________________
    44
    45
    46  #!/bin/bash
    47  ####################################
    48  #                      #
    49  # START                #
    50  # g0ne_@p0stal                     #
    51  # ImPerialTeKSolutions@outlook.com #
    52  #                  #
    53  ####################################
    54
    55  #+ ITS_ URL Video DL: batch version 1.0
    56  #* Batch download part of program
    57
    58  needed_tool='youtube-dl yad kdialog ffmpeg'
    59  require_tools ()
    60  {
    61      local NOT_AVAIL=""
    62      for TOOL in $needed_tool; do
    63          if [ "`which $TOOL 2> /dev/null`" == "" ]; then NOT_AVAIL="$NOT_AVAIL $TOOL"; fi
    64      done
    65      if [[ "$NOT_AVAIL" != "" ]]; then
    66          echo "ERROR: The following required tool(s) cannot be found: $NOT_AVAIL"
    67          exit 3
    68      fi
    69  }
    70
    71  # Check If Package Downloaded
    72  require_tools
    73
    74  # Dimensions
    75  height=200
    76  width=500
    77
    78  # Numerical batch list's
    79  yad --height=$height --width=$width --title "Numerical Answer batch#.list" --entry > /tmp/batch#.$$
    80  _number=`cat /tmp/batch#.$$`
    81
    82  # Download Of Files Location
    83  cd "$(yad --title "Change Directory To Download Destination" --file-selection --directory)"
    84
    85  # Batch list
    86  batch_list=batch$_number.list
    87
    88  # Start Program Here
    89  time youtube-dl --ignore-config --hls-prefer-native --add-metadata -i -c --yes-playlist --external-downloader aria2c -a $HOME/$batch_list  | pv -tpeba -s 100M
    90  ding
    91
    92  # Cleanup
    93  rm $HOME/$batch_list
    94
    95  # Exit
    96  exit $?
    97
    98
    99  ---------------------------------------------
   100
   101
   102  Sun Sep  2 06:38:02 EDT 2018
   103  NOTE:
   104
   105  LINUX ESSENTIALS
   106  [ Workshops ]       [ R_&_BioC ]        [ BioC-Seq ]        [ R_Programming ]       [ EMBOSS ]      [ Linux ]       [ Cluster ]
   107  Author: Thomas Girke, UC Riverside
   108  New version of this manual
   109
   110  Index
   111  INTRODUCTION
   112  BASICS
   113  UNIX HELP
   114  FINDING THINGS
   115  PERMISSIONS & OWNERSHIP
   116  USEFUL COMMANDS
   117  JOB/PROCESS MANAGEMENT
   118  TEXT VIEWING
   119  TEXT EDITORS
   120  THE UNIX SHELL
   121  SIMPLE SHELL ONE-LINER SCRIPTS
   122  SIMPLE PERL ONE-LINER SCRIPTS
   123  REMOTE COPY
   124  ARCHIVING AND COMPRESSING
   125  SIMPLE INSTALLS
   126  DEVICES
   127  ENVIRONMENT VARIABLES
   128  EXERCISES
   129  INTRODUCTION
   130  Why UNIX?
   131  Multitasking
   132  Remote tasking ("real networking")
   133  Multiuser
   134  Access to shell, programming languages, databases, open-source projects
   135  Better performance, less expensive (free), more up-to-date
   136  Many more reasons
   137  How to get access
   138  Installation on local machine (not required!!!)
   139  Get account on bioinfo.ucr.edu server:
   140  email tgirke@citrus.ucr.edu
   141  requirements for Windows and Mac OS X: http://faculty.ucr.edu/~tgirke/GCG.htm#General
   142  UNIX variants
   143  UNIX: Solaris, IRIX, HP-UX, Tru64-UNIX, Free's, LINUX, ...
   144  LINUX distributions
   145  RedHat, Debian, Mandrake, Caldera, Slackware, SuSE, ...
   146
   147  BASICS
   148  Syntax for this manual
   149  Remember the UNIX/LINUX command line is case sensitive!
   150  "$" indicates start of command
   151  "#" indicates end of command and start of comment
   152  The text in green or red monospace font represents the actual command. The "$" and "#" symbols are not part of it. The commands in red emphasize essential information for beginners.
   153  "<...>" or "my_..." refers to variables and file names that need to be specified by the user. The arrows "<...>" need to be excluded, because they are generic UNIX redirection functions!
   154  Login from Windows:
   155  open PuTTY and select ssh
   156  provide host name (IP) and session name
   157
   158  $ user name: ...
   159  $ password: ...
   160  Setup for graphics emulation
   161  WinSCP for file exchange
   162  Login from Mac OS-X or LINUX
   163  open terminal and type:
   164
   165  $ ssh <user_name>@<host name(IP)>
   166  $ user name: ...
   167  $ password: ...
   168  Changing password:
   169  $ passwd # follow instructions
   170  Orientation
   171  $ pwd # present working directory
   172  $ ls # content of pwd
   173  $ ll # similar as ls, but provides additional info on files and directories
   174  $ ll -a # includes hidden files (.name) as well
   175  $ ll -R # lists subdirectories recursively
   176  $ ll -t # lists files in chronological order
   177  $ stat <file_name> # provides all attributes of a file
   178  $ whoami # shows as who you are logged in
   179  $ hostname # shows on which machine you are
   180  Files and directories
   181  $ mkdir <dir_name> # creates specified directory
   182  $ cd <dir_name> # switches into specified directory
   183  $ cd .. # moves one directory up
   184  $ cd ../../ # moves two directories up (and so on)
   185  $ cd # brings you to highest level of your home directory
   186  $ rmdir <dir_name> # removes empty directory
   187  $ rm <file_name> # removes file name
   188  $ rm -r <dir_name> # removes directory including its content, but asks for confirmation, 'f' argument turns confirmation off
   189  $ mv <name1> <name2> # renames directories or files
   190  $ mv <name> <path> # moves file/directory as specified in path
   191  $ cp <name> <path> # copy file/directory as specified in path (-r to include content in directories)
   192  Copy and paste
   193  Depends on local environment. Usually one of the following methods works:
   194
   195  Copy: Ctrl&Shift&c or right/middle mouse click
   196  Paste: Ctrl&Shift&p or right/middle mouse click
   197  Handy shortcuts
   198  $ . # refers to pwd
   199  $ ~/ # refers to user's home directory
   200  $ history # shows all commands you have used recently
   201  $ !<number> # starts an old command by providing its ID number
   202  $ up(down)_key # scrolls through command history
   203  $ <incomplete path/file_name> TAB # completes path/file_name
   204  $ <incomplete command> SHIFT&TAB # completes command
   205  $ Ctrl a # cursor to beginning of command line
   206  $ Ctrl e # cursor to end of command line
   207  $ Ctrl d # delete character under cursor
   208  $ Ctrl k # delete line from cursor, content goes into kill buffer
   209  $ Ctrl y # paste content from Ctrl k
   210
   211  UNIX HELP
   212  $ man # general help
   213  $ man wc # manual on program 'word count' wc
   214  $ wc --help # short help on wc
   215  $ info wc # more detailed information system (GNU)
   216  $ apropos wc # retrieves pages where wc appears
   217  Online help: SuperMan Pages, Linux Documentation Project (LDP)
   218
   219  FINDING THINGS
   220  Finding files, directories and applications
   221  $ find -name "*pattern*" # searches for *pattern* in and below current directory
   222  $ find /usr/local -name "*blast*" # finds file names *blast* in specfied directory
   223  $ find /usr/local -iname "*blast*" # same as above, but case insensitive
   224  additional useful arguments: -user <user name>, -group <group name>, -ctime <number of days ago changed>
   225  $ find ~ -type f -mtime -2 # finds all files you have modified in the last two days
   226  $ locate <pattern> # finds files and dirs that are written into update file
   227  $ which <application_name> # location of application
   228  $ whereis <application_name> # searches for executeables in set of directories
   229  $ dpkg -l | grep mypattern # find Debian packages and refine search with grep pattern
   230  Finding things in files
   231  $ grep pattern file # provides lines in 'file' where pattern 'appears', if pattern is shell function use single-quotes: '>'
   232  $ grep -H pattern # -H prints out file name in front of pattern
   233  $ grep 'pattern' file | wc # pipes lines with pattern into word count wc (see chapter 8); wc arguments: -c: show only bytes, -w: show only words, -l: show only lines; help on regular expressions: $ man 7 regex or man perlre
   234  $ find /home/my_dir -name '*.txt' | xargs grep -c ^.* # counts line numbers on many files and records each count along with individual file name; find and xargs are used to circumvent the Linux wildcard limit to apply this function on thousands of files.
   235
   236  PERMISSIONS & OWNERSHIP
   237  How does it work
   238  $ ls -al # shows something like this for each file/dir: drwxrwxrwx
   239  d: directory
   240  rwx: read write execute
   241  first triplet: user permissions (u)
   242  second triplet: group permissions (g)
   243  third triplet: world permissions (o)
   244  To assign write and execute permissions to user and group:
   245  $ chmod ug+rx my_file
   246  To remove all permissions from all three user groups:
   247  $ chmod ugo-rwx my_file
   248  '+' causes the permissions selected to be added
   249  '-' causes them to be removed
   250  '=' causes them to be the only permissions that the file has.
   251  Example for number system:
   252  $ chmod +rx public_html/ or $ chmod 755 public_html/
   253  Change ownership
   254  $ chown <user> <file or dir> # changes user ownership
   255  $ chgrp <group> <file or dir> # changes group ownership
   256  $ chown <user>:<group> <file or dir> # changes user & group ownership
   257
   258  USEFUL UNIX COMMANDS
   259  $ df # disk space
   260  $ free # memory info
   261  $ uname -a # shows tech info about machine
   262  $ bc # command-line calculator (to exit type 'quit')
   263  $ wget ftp://ftp.ncbi.nih.... # file download from web
   264  $ /sbin/ifconfig # give IP and other network info
   265  $ ln -s original_filename new_filename # creates symbolic link to file or directory
   266  $ du -sh # displays disk space usage of current directory
   267  $ du -sh * # displays disk space usage of individual files/directories
   268  $ du -s * | sort -nr # shows disk space used by different directories/files sorted by size
   269
   270  JOB/PROCESS MANAGEMENT
   271  $ who # shows who is logged into system
   272  $ w # shows which users are logged into system and what they are doing
   273  $ ps # shows processes running by user
   274  $ ps -e # shows all processes on system; try also '-a' and '-x' arguments
   275  $ ps aux | grep <user_name> # shows all processes of one user
   276  $ top # view top consumers of memory and CPU
   277  $ mtop # displays multicomputer/CPU processes
   278  $ Ctrl z <enter> bg or fg <enter> # suspends a process to bring into back- or foreground
   279  $ Ctrl c # stops an initiated process
   280  $ kill <ps_ID> # Kills specified job; if this doesn't do it, add -9 as argument. Also, type <%1> then <CTRL C>.
   281  $ renice -n <priority_value> # change priority value, which range from 1-19, the higher the value the lower the priority, default is 10
   282
   283  TEXT VIEWING
   284  $ less <my_file> # more versatile text viewer than 'more', 'G' moves to end of text, 'g' to beginning, '/' find forward, '?' find backwards
   285  $ more <my_file> # views text, use space bar to browse, hit 'q' to exit
   286  $ cat <my_file> # concatenates files and prints content to standard output
   287
   288  TEXT EDITORS
   289  VI and VIM
   290  Non-graphical (terminal-based) editor. Vi is guaranteed to be available on any system. Vim is the improved version of vi.
   291  EMACS
   292  Window-based editor. You still need to know keystroke commands to use it. Installed on all Linux distributions and on most other Unix systems.
   293  XEMACS
   294  More sophisticated version of emacs, but usually not installed by default. All common commands are available from menus. Very powerful editor, with built-in syntax checking, Web-browsing, news-reading, manual-page browsing, etc.
   295  PICO
   296  Simple terminal-based editor available on most versions of Unix. Uses keystroke commands, but they are listed in logical fashion at bottom of screen.
   297  VIM MANUAL (essentials marked in red)
   298  BASICS
   299  $ vim my_file_name # open/create file with vim
   300  $ i # INSERT MODE
   301  $ ESC # NORMAL (NON-EDITING) MODE
   302  $ : # commands start with ':'
   303  $ :w # save command; if you are in editing mode you have to hit ESC first!!
   304  $ :q # quit file, don't save
   305  $ :q! # exits WITHOUT saving any changes you have made
   306  $ :wq # save and quit
   307  $ R # replace MODE
   308  $ r # replace only one character under cursor
   309  $ q: # history of commands (from NORMAL MODE!), to reexecute one of them, select and hit enter!
   310  $ :w new_filename # saves into new file
   311  $ :#,#w new_filename # saves specific lines (#,#) to new file
   312  $ :# go to specified line number
   313  HELP
   314  $ Useful list of vim commands: Vim Commands Cheat Sheet, VimCard, Vim Basics
   315  $ vimtutor # open vim tutorial from shell
   316  $ :help # opens help within vim, hit :q to get back to your file
   317  $ :help <topic> # opens help on specified topic
   318  $ |help_topic| CTRL-] # when you are in help this command opens help topic specified between |...|, CTRL-t brings you back to last topic
   319  $ :help <topic> CTRL-D # gives list of help topics that contain key word
   320  $ : <up-down keys> # like in shell you get recent commands!!!!
   321  MOVING AROUND IN FILE
   322  $ $ # moves cursor to end of line
   323  $ A # same as $, but switches to insert mode
   324  $ 0 (zero) # moves cursor to beginning of line
   325  $ CTRL-g # shows at status line filename and the line you are on
   326  $ SHIFT-G # brings you to bottom of file, type line number (isn't displayed) then SHIFT-G # brings you to specified line#
   327  DISPLAY
   328  WRAPPING AND LINE NUMBERS
   329  $ :set nowrap # no word wrapping, :set wrap # back to wrapping
   330  $ :set number # shows line numbers, :set nonumber # back to no-number mode
   331  WORKING WITH MANY FILES & SPLITTING WINDOWS
   332  $ vim *.txt # opens many files at once; ':n' switches between files
   333  $ :wall or :qall # write or quit all open files
   334  $ vim -o *.txt # opens many files at once and displays them with horizontal split, '-O' does vertical split
   335  $ :args *.txt # places all the relevant files in the argument list $ :all # splits all files in the argument list (buffer) horizontally $ CTRL-w # switch between windows
   336  $ :split # shows same file in two windows
   337  $ :split <file-to-open> # opens second file in new window
   338  $ :vsplit # splits windows vertically, very useful for tables, ":set scrollbind" let's you scroll all open windows symultaneously
   339  $ :close # closes current window
   340  $ :only # closes all windows except current one
   341  SPELL CHECKING & Dictionary
   342  $ aspell -c <file> # shell command
   343  $ aspell -l <my_file> # shell command
   344  $ :! dict <word> # meaning of word
   345  $ :! wn 'word' -over # synonyms of word
   346  PRINTING FILE
   347  $ :ha # prints entire file
   348  $ :#,#ha # prints specified lines: #,#
   349  MERGING/INSERTING FILES
   350  $ :r <filename> # inserts content of specified file after cursor
   351  UNDO/REDO
   352  $ u # undo last command
   353  $ U # undo all changes on current line
   354  $ CTRL-R # redo one change which was undone
   355  DELETION/CUT (switch to NORMAL mode)
   356  $ x # deletes what is under cursor
   357  $ dw # deletes from curser to end of word including the space
   358  $ de # deletes from curser to end of word NOT including the space
   359  $ cw # deletes rest of word and lets you then insert, hit ESC to continue with NORMAL mode
   360  $ c$ # deletes rest of line and lets you then insert, hit ESC to continue with with NORMAL mode
   361  $ d$ # deletes from cursor to the end of the line
   362  $ dd # deletes entire line
   363  $ 2dd # deletes next two lines, continues: 3dd, 4dd and so on.
   364  PUT (PASTE)
   365  $ p # uses what was deleted/cut and pastes it behind cursor
   366  COPY & PASTE
   367  $ yy # copies line, for copying several lines do 2yy, 3yy and so on
   368  $ p # pastes clipboard behind cursor
   369  SEARCH IN FILE (most regular expressions work)
   370  $ /my_pattern # searches for my_pattern downwards, type n for next match
   371  $ ?my_pattern # seraches for my_pattern upwards, type n for next match
   372  $ :set ic # switches to ignore case search (case insensitive)
   373  $ :set hls # switches to highlight search (highlights search hits)
   374  REPLACE WITH REGULAR EXPRESSIONS (great intro: A Tao of Regular Expressions)
   375  $ :s/old_pat/new_pat/ # replaces first occurence in a line
   376  $ :s/old_pat/new_pat/g # replaces all occurence in a line
   377  $ :s/old_pat/new_pat/gc # add 'c' to ask for confirmation
   378  $ :#,#s/old_pat/new_pat/g # replaces all occurence between line numbers: #,#
   379  $ :%s/old_pat/new_pat/g # replaces all occurence in file
   380  $ :%s/\(pattern1\)\(pattern2\)/\1test\2/g # regular expression to insert, you need here '\' in front of parentheses (<# Perl)
   381  $ :%s/\(pattern.*\)/\1 my_tag/g # appends something to line containing pattern (<# .+ from Perl is .* in VIM)
   382  $ :%s/\(pattern\)\(.*\)/\1/g # removes everything in lines after pattern
   383  $ :%s/\(At\dg\d\d\d\d\d\.\d\)\(.*\)/\1\t\2/g # inserts tabs between At1g12345.1 and Description
   384  $ :%s/\n/new_pattern/g #Replaces return signs
   385  $ :%s/pattern/\r/g #Replace pattern with return signs!!
   386  $ :%s/\(\n\)/\1\1/g # insert additional return signs
   387  $ :%s/\(^At\dg\d\d\d\d\d.\d\t.\{-}\t.\{-}\t.\{-}\t.\{-}\t\).\{-}\t/\1/g # replaces content between 5th and 6th tab (5th column), '{-}' turns off 'greedy' behavior
   388  $ :#,#s/\( \{-} \|\.\|\n\)/\1/g # performs simple word count in specified range of text
   389  $ :%s/\(E\{6,\}\)/<font color="green">\1<\/font>/g # highlight pattern in html colors, here highlighting of >= 6 occurences of Es
   390  $ :%s/\([A-Z]\)/\l\1/g # change uppercase to lowercase, '%s/\([A-Z]\)/\u\1/g' does the opposite
   391  $ :g/my_pattern/ s/\([A-Z]\)/\l\1/g | copy $ # uses 'global' command to apply replace function only on those lines that match a certain pattern. The 'copy $' command after the pipe '|' prints all matching lines at the end of the file.
   392  $ :args *.txt | all | argdo %s/\old_pat/new_pat/ge | update # Command 'args' places all relevant files in the argument list (buffer); 'all' displays each file in separate split window; command 'argdo' applies replacement to all files in argument list (buffer); flag 'e' is necessary to avoid stop at error messages for files with no matches; command 'update' saves all changes to files that were updated.
   393  MATCHING PARENTHESES SEARCH
   394  - place curser on (, [ or { and type % # curser moves to matching parentheses
   395  HTML EDITING
   396  -Convert text file to html format:
   397  $ :runtime! syntax/2html.vim # run this command with open file in Vim
   398  SHELL COMMAND IN VIM
   399  $ :!<SHELL_COMMAND> <ENTER> # executes any shell command, hit <enter> to return
   400  $ :sh # switches window to shell, 'exit' switches back to vim
   401  USING VIM AS TABLE EDITOR
   402  $ v # starts visual mode for selecting characters
   403  $ V # starts visual mode for selecting lines
   404  $ CTRL-V # starts visual mode for selecting blocks (use CTRL-q in gVim under Windows). This allows column-wise selections and operations like inserting and deleting columns. To restrict substitude commands to a column, one can select it and switch to the command-line by typing ':'. After this the substitution sytax for a selected block looks like this: '<,'>s///.
   405  $ :set scrollbind # starts simultaneous scrolling of 'vsplitted' files. To set to horizontal binding of files, use command ':set scrollopt=hor' (after first one). Run all these commands before the ':split' command.
   406  $ :AlignCtrl I= \t then :%Align # This allows to align tables by column separators (here '\t') when the Align utility from Charles Campbell's is installed.
   407  To sort table rows by selected lines or block, perform the visual select and then hit F3 key. The rest is interactive. To enable this function one has to include in the .vimrc file from Gerald Lai the Vim sort script.
   408  MODIFY VIM SETTINGS (in file .vimrc)
   409  - see last chapter of vimtutor (start from shell)
   410  - useful .vimrc sample
   411  - when vim starts to respond very slowly then one may need to delete the .viminf* files in home directory
   412
   413  THE UNIX SHELL
   414  When you log into UNIX/LINUX the system starts a program called SHELL. It provides you with a working environment and interface to the operating system. Usually there are many different shell programs installed.
   415  $ finger <user_name> # shows which shell you are using
   416  $ chsh -l # gives list of shell programs available on your system (does not work on all UNIX variants)
   417  $ <shell_name> # switches to different shell
   418  STDIN, STDOUT, STDERR, REDIRECTORS, OPERATORS & WILDCARDS (more on this @ LINUX HOWTOs)
   419  By default, many UNIX commands read from standard input (STDIN) and send their output to standard out (STDOUT). You can redirect them by using the following commands:
   420  $ file* # * is wildcard to specify many files
   421  $ ls > file # prints ls output into specified file
   422  $ command < my_file # uses file after '<' as STDIN
   423  $ command >> my_file # appends output of one command to file
   424  $ command | tee my_file # writes STDOUT to file and prints it to screen; alternative way to do this:
   425  $ command > my_file; cat my_file
   426  $ command > /dev/null # turns off progress info of applications by redirecting their output to /dev/null
   427  $ grep my_pattern my_file | wc # Pipes (|) output of 'grep' into 'wc'
   428  $ grep my_pattern my_non_existing_file 2 > my_stderr # prints STDERR to file
   429  Useful shell commands
   430  $ cat <file1> <file2> > <cat.out> # concatenate files in output file 'cat.out'
   431  $ paste <file1> <file2> > <paste.out> # merges lines of files and separates them by tabs (useful for tables)
   432  $ cmp <file1> <file2> # tells you whether two files are identical
   433  $ diff <fileA> <fileB> # finds differences between two files
   434  $ head -<number> <file> # prints first lines of a file
   435  $ tail -<number> <file> # prints last lines of a file
   436  $ split -l <number> <file> # splits lines of file into many smaller ones
   437  $ csplit -f out fasta_batch "%^>%" "/^>/" "{*}" # splits fasta batch file into many files at '>'
   438  $ sort <file> # sorts single file, many files and can merge (-m) them, -b ignores leading white space, ...
   439  $ sort -k 2,2 -k 3,3n input_file > output_file # sorts in table column 2 alphabetically and column 3 numerically, '-k' for column, '-n' for numeric
   440  $ sort input_file | uniq > output_file # uniq command removes duplicates and creates file/table with unique lines/fields
   441  $ join -1 1 -2 1 <table1> <table2> # joins two tables based on specified column numbers (-1 file1, 1: col1; -2: file2, col2). It assumes that join fields are sorted. If that is not the case, use the next command:
   442  $ sort table1 > table1a; sort table2 > table2a; join -a 1 -t "`echo -e '\t'`" table1a table2a > table3 # '-a <table>' prints all lines of specified table! Default prints only all lines the two tables have in common. '-t "`echo -e '\t'`" ->' forces join to use tabs as field separator in its output. Default is space(s)!!!
   443  $ cat my_table | cut -d , -f1-3 # cut command prints only specified sections of a table, -d specifies here comma as column separator (tab is default), -f specifies column numbers.
   444  $ grep and egrep # see chapter 4
   445
   446  SIMPLE SHELL ONE-LINER SCRIPTS
   447  Useful One-Liners (script download)
   448  $ for i in *.input; do mv $i ${i/name\.old/name\.new}; done # renames file name.old to name.new
   449  - To test things first, insert 'echo' between 'do mv' (above).
   450  $ for i in *.input; do ./application $i; done # runs application in loops on many input files
   451  $ for i in *.input; do fastacmd -d /data/../database_name -i $i > $i.out; done # runs fastacmd in loops on many *.input files and creates *.out files
   452  $ for i in *.pep; do target99 -db /usr/../database_name -seed $i -out $i; done # runs SAM's target99 on many input files
   453  $ for j in 0 1 2 3 4 5 6 7 8 9; do grep -iH <my_pattern> *$j.seq; done # searches in > 10,000 files for pattern and prints occurences together with file names.
   454  $ for i in *.pep; do echo -e "$i\n\n17\n33\n\n\n" | ./tmpred $i > $i.out; done # example of how to run an interactive application (tmpred) that asks for file name input/output
   455  $ for i in *.fasta1; do blast2 -p blastp -i $i -j ${i/_*fasta1/_*fasta2} >> my_out_file; done # runs BLAST2 for all *.fasa1/*.fasta2 file pairs in the order specified by file names and writes results into one file. This example uses two variables in a for loop. The content of the second variable gets specified in each loop by a replace function.
   456  $ for i in *.fasta; do for j in *.fasta; do blast2 -p blastp -F F -i $i -j $j >> my_out_file; done; done; # runs BLAST2 in all-against-all mode and writes results into one file; '-F F' turns low-complexity filter off
   457  How to write a script
   458  - create file which contains in first line:
   459  #!/bin/bash
   460  - place shell commands in file
   461  - run <chmod +x my_shell_script> to make it executable
   462  - run shell script like this: ./my_shell_script
   463  - when you place it into /usr/local/bin you only type its name from any user account
   464
   465  SIMPLE PERL ONE-LINER SCRIPTS
   466  Useful One-Liners
   467  $ perl -p -i -w -e 's/pattern1/pattern2/g' input_file # replace something (e.g. return signs) in file using regular expressions; use $1 to backreference to pattern placed in parentheses
   468  '-p' lets perl know to write program; '-i.bak' creates backup file *.bak, only -i doesn't; '-w' turns on warnings; '-e' executeable code follows
   469  $ perl -ne 'print if (/my_pattern1/ ? ($c=1) : (--$c > 0)) ; print if (/my_pattern2/ ? ($d = 1) : (--$d > 0))' my_infile > my_outfile # parses lines that contain pattern1 and pattern2
   470  following lines after pattern can be specified in '$c=1' and '$d=1'; for OR function use this syntax: '/(pattern1|pattern2)/'
   471
   472  REMOTE COPY: WGET, SCP and NCFTP
   473  WGET (file download from the www)
   474  $ wget ftp://ftp.ncbi.nih.... # file download from www; add option '-r' to download entire directories
   475  SCP (secure copy between machines)
   476  General syntax
   477  $ scp source target # Use form 'userid@machine_name' if your local and remote user ids are differnt. If they are the same you can use only 'machine_name'.
   478
   479  Examples
   480  Copy file from Server to Local Machine (type from local machine prompt):
   481  $ scp user@remote_host:file.name . # '.' copies to pwd, you can specify here any directory, use wildcards to copy many files at once.
   482  Copy file from Local Machine to Server:
   483  $ scp file.name user@remote_host:~/dir/newfile.name
   484  Copy entire directory from Server to Local Machine (type from local machine prompt):
   485  $ scp -r user@remote_host:directory/ ~/dir
   486  Copy entire directory from Local Machine to Server (type from local machine prompt):
   487  $ scp -r directory/ user@remote_host:directory/
   488  Copy between two remote hosts (e.g. from bioinfo to cache):
   489  similar as above, just be logged in one of the remote hosts:
   490  $ scp -r directory/ user@remote_host:directory/
   491  NICE FTP
   492  $ open ncftp
   493  $ ncftp> open ftp.ncbi.nih.gov
   494  $ ncftp> cd /blast/executables
   495  $ ncftp> get blast.linux.tar.Z (skip extension: @)
   496  $ ncftp> bye
   497
   498  ARCHIVING AND COMPRESSING
   499  Archiving and compressing
   500  $ tar -cvf my_file.tar mydir/ # Builds tar archive of files or directories. For directories, execute command in parent directory. Don't use absolute path.
   501  $ tar -czvf my_file.tgz mydir/ # Builds tar archive with compression of files or directories. For directories, execute command in parent directory. Don't use absolute path.
   502  Viewing Archives
   503  $ tar -tvf my_file.tar
   504  $ tar -tzvf my_file.tgz
   505  Extracting
   506  $ tar -xvf my_file.tar
   507  $ tar -xzvf my_file.tgz
   508  $ gunzip my_file.tar.gz # or unzip my_file.zip, uncompress my_file.Z, or bunzip2 for file.tar.bz2
   509  $ find -name '*.zip' | xargs -n 1 unzip # this command usually works for unziping many files that were compressed under Windows
   510  try also:
   511  $ tar zxf blast.linux.tar.Z
   512  $ tar xvzf file.tgz
   513  options:
   514  f: use archive file
   515  p: preserve permissions
   516  v: list files processed
   517  x: exclude files listed in FILE
   518  z: filter the archive through gzip
   519
   520  SIMPLE INSTALLS
   521  Systems-wide installations
   522  Installations for systems-wide usage are the responsibility of system administrator
   523  To find out if an application is installed, type:
   524  $ which <application_name>
   525  $ whereis <application_name> # searches for executeables in set of directories, doesn't depend on your path
   526  Most applications are installed in /usr/local/bin or /usr/bin. You need root permissions to write to these directories.
   527  Perl scripts go into /usr/local/bin, Perl modules (*.pm) into /usr/local/share/perl/5.8.0/. To copy executables in one batch, use command: cp `find -perm -111 -type f` /usr/local/bin
   528  Applications in user accounts
   529  Create a new directory, download application into this directory, unpack it (see chapter 13) and follow package-specific installation instructions.
   530  Usually you can then already run this application when you specify its location e.g.: /home/user/my_app/blastall.
   531  If you want you can add this directory to your PATH by typing from this directory:
   532  $ PATH=.:$PATH; export PATH # this allows you to run application by providing only its name; when you do echo $PATH you will see .: added to PATH.
   533  Intstallation of RPMs
   534  $ rpm -i application_name.rpm
   535  To check which version of RPM package is installed, type:
   536  $ rpm --query <package_name>
   537  Help and upgrade files for RPMs can be found at http://rpmfind.net/.
   538  Installation of Debian packages
   539  Check whether your application is available at: http://www.debian.org/intro/about, then you type (no download):
   540  $ apt-cache search phylip #searches for application "phylip" from command line
   541  $ apt-cache show phylip #provides description of program
   542  $ apt-get install phylip # example for phylip install, manuals can be found in /usr/doc/phylip/, use zless or lynx to read documentation (don't unzip).
   543  $ apt-get update # do once a month do update Debian packages
   544  $ apt-get upgrade -u # to upgrade after update from above
   545  $ dpkg -i # install data package from local package file (e.g. after download)
   546  $ aptitude # Debian package manageing interface (Ctrl-t opens menues)
   547  $ aptitude search vim # search for packages on system and in Debian depositories
   548
   549  DEVICES
   550  Mount/unmount usb/floppy/cdrom
   551  $ mount /media/usb
   552  $ umount /media/usb
   553  $ mount /media/cdrom
   554  $ eject /media/cdrom
   555  $ mount /media/floppy
   556
   557  ENVIRONMENT VARIABLES
   558  $ xhost user@host # adds X permissions for user on server.
   559  $ echo DISPLAY # shows current display settings
   560  $ export (setenv) DISPLAY=<local_IP>:0 # change environment variable
   561  $ unsetenv DISPLAY # removes display variable
   562  $ printenv # prints all environment variables
   563  $ $PATH # list of directories that the shell will search when you type a command
   564  - You can edit your default DISPLAY setting for your account by adding it to file .bash_profile.
   565
   566  EXERCISES
   567  Exercise 1
   568  Download proteome of Halobacterium spec. from ftp://ftp.ncbi.nih.gov/genbank/genomes/Bacteria/Halobacterium_sp/AE004437.faa (use wget or web browser for download)
   569  How many predicted proteins are there?
   570  $ grep '>' AE004437.faa | wc
   571  How many proteins contain the pattern "WxHxxH[1-2]"?
   572  $ egrep 'W.H..H{1,2}' AE004437.faa | wc
   573  Use the find function (/) in 'less' to fish out the proteins containing this pattern or more elegantly do it with awk:
   574  $ awk --posix -v RS='>' '/W.H..(H){1,2}/ { print ">" $0;}' AE004437.faa | less
   575  Create a BLASTable database with formatdb
   576  $ formatdb -i AE004437.faa -p T -o T
   577  '-p F' for nucleotide and '-p T' for protein databases
   578  Generate list of sequence IDs for above pattern match result and retrieve its sequences with fastacmd from formatted database
   579  $ fastacmd -d AE004437.faa -i my_IDs > seq
   580  Generate several lists of sequence IDs from various pattern match results and retrieve their sequences in one step using the fastacmd in for loop
   581  $ for i in *.my_ids; do fastacmd -d AE004437.faa -i $i > $i.out; done
   582  Run blastall with a few proteins against newly created database or against Halobacterium or UniProt database (/data/UNIPROT/blast/uniprot)
   583  $ blastall -p blastp -i input.file -d AE004437.faa -o blastp.out -e 1e-6 -v 10 -b 10 &
   584  Parse blastall output into Excel spread sheet:
   585  a) using biocore parser
   586  $ blastParse -c <hits> -i <blast.out> -o <blast.parse>
   587  b) using BioPerl parser
   588  $ bioblastParse.pl blast.out
   589  Run HMMPFAM search with above proteins against Pfam database
   590  $ hmmpfam -E 0.1 --acc -A0 /data/PFAM/Pfam_ls input.file > output.pfam
   591  Parse result with BioPerl parser
   592  $ hmmSummary output.pfam > hmm.summary
   593  Exercise 2
   594  Split sample fasta batch file with csplit (use sequence file from exercise 1).
   595  Concatenate single fasta files from (1) to one batch file.
   596  BLAST two related sequences, retrieve the result in table format and use join to identify common hit IDs in the two tables.
   597  Exercise 3
   598  write a shell script that executes several BLAST searches at once:
   599  #!/bin/sh
   600  blastall -p blastp -d /.../my_database -i /.../my_input -o my_out -e 1e-6 -v 10 -b 10 &
   601  blastall -p blastp -d /.../my_database -i /.../my_input -o my_out -e 1e-6 -v 10 -b 10 &
   602  Exercise 4
   603  Create multiple alignment with ClustalW (e.g. use sequences with 'W.H..HH' pattern)
   604  $ clustalw my_fasta_batch
   605  Exercise 5
   606  Reformat alignment into PHYILIP format using 'seqret' from EMBOSS
   607  $ seqret clustal::my_align.aln phylip::my_align.phylip
   608  Exercise 6
   609  Create neighbor-joining tree with PHYLIP
   610  $ cp my_align.phylip infile
   611  $ phylip protdist # creates distance matrix
   612  $ cp outfile infile
   613  $ phylip neighbor # use default settings
   614  $ cp outtree intree
   615  $ phylip retree # displays tree and can use midpoint method for defining root of tree, my typical command sequence is: 'N' 'Y' 'M' 'W' 'R' 'R' 'X'
   616  $ cp outtree my_tree.dnd
   617  View your tree in TreeBrowse or open it in TreeView
   618
   619  ---------------------------------------------
   620
   621  prompt () {
   622      local -a prompt_opts theme_active
   623      zstyle -g theme_active :prompt-theme cleanup || {
   624          [[ -o promptbang ]] && prompt_opts+=(bang)
   625          [[ -o promptcr ]] && prompt_opts+=(cr)
   626          [[ -o promptpercent ]] && prompt_opts+=(percent)
   627          [[ -o promptsp ]] && prompt_opts+=(sp)
   628          [[ -o promptsubst ]] && prompt_opts+=(subst)
   629          zstyle -e :prompt-theme cleanup 'zstyle -d :prompt-theme cleanup;' 'prompt_default_setup;' ${PS1+PS1="${(q)PS1}"} ${PS2+PS2="${(q)PS2}"} ${PS3+PS3="${(q)PS3}"} ${PS4+PS4="${(q)PS4}"} ${RPS1+RPS1="${(q)RPS1}"} ${RPS2+RPS2="${(q)RPS2}"} ${RPROMPT+RPROMPT="${(q)RPROMPT}"} ${RPROMPT2+RPROMPT2="${(q)RPROMPT2}"} ${PSVAR+PSVAR="${(q)PSVAR}"} "precmd_functions=(${(q)precmd_functions[@]})" "preexec_functions=(${(q)preexec_functions[@]})" "prompt_opts=( ${prompt_opts[*]} )" 'reply=(yes)'
   630      }
   631      set_prompt "$@"
   632      (( ${#prompt_opts} )) && setopt noprompt{bang,cr,percent,sp,subst} "prompt${^prompt_opts[@]}"
   633      true
   634  }
   635  clippaste () {
   636      emulate -L zsh
   637      if [[ $OSTYPE == darwin* ]]
   638      then
   639          pbpaste
   640      elif [[ $OSTYPE == cygwin* ]]
   641      then
   642          cat /dev/clipboard
   643      else
   644          if (( $+commands[xclip] ))
   645          then
   646              xclip -out -selection clipboard
   647          elif (( $+commands[xsel] ))
   648          then
   649              xsel --clipboard --output
   650          else
   651              print "clipcopy: Platform $OSTYPE not supported or xclip/xsel not installed" >&2
   652              return 1
   653          fi
   654      fi
   655  }
   656
   657  Sun Sep  2 12:50:36 EDT 2018
   658  NOTE:
   659
   660  In the example below we will print the numbers 1 through to 10:
   661
   662  while_loop.sh
   663  #!/bin/bash
   664  # Basic while loop
   665  counter=1
   666  while [ $counter -le 10 ]
   667  do
   668  echo $counter
   669  ((counter++))
   670  done
   671  echo All done
   672
   673
   674  Let's break it down:
   675
   676
   677
   678  ---------------------------------------------
   679
   680
   681  For Loops
   682  The for loop is a little bit different to the previous two loops. What it does is say for each of the items in a given list, perform the given set of commands. It has the following syntax.
   683
   684  for var in <list>
   685  do
   686  <commands>
   687  done
   688
   689  The for loop will take each item in the list (in order, one after the other), assign that item as the value of the variable var, execute the commands between do and done then go back to the top, grab the next item in the list and repeat over.
   690
   691  The list is defined as a series of strings, separated by spaces.
   692
   693  Here is a simple example to illustrate:
   694
   695  for_loop.sh
   696  #!/bin/bash
   697  # Basic for loop
   698  names='Stan Kyle Cartman'
   699  for name in $names
   700  do
   701  echo $name
   702  done
   703  echo All done
   704  Let's break it down:
   705
   706  Line 4 - Create a simple list which is a series of names.
   707  Line 6 - For each of the items in the list $names assign the item to the variable $name and do the following commands.
   708  Line 8 - echo the name to the screen just to show that the mechanism works. We can have as many commands here as we like.
   709  Line 11 - echo another command to show that the bash script continued execution as normal after all the items in the list were processed.
   710  ./for_loop.sh
   711  Stan
   712  Kyle
   713  Cartman
   714  All done
   715  Ranges
   716  We can also process a series of numbers
   717
   718  for_loop_series.sh
   719  #!/bin/bash
   720  # Basic range in for loop
   721  for value in {1..5}
   722  do
   723  echo $value
   724  done
   725  echo All done
   726  Line 4 - It's important when specifying a range like this that there are no spaces present between the curly brackts { }. If there are then it will not be seen as a range but as a list of items.
   727  ./for_loop_series.sh
   728  1
   729  2
   730  3
   731  4
   732  5
   733  All done
   734  When specifying a range you may specify any number you like for both the starting value and ending value. The first value may also be larger than the second in which case it will count down.
   735
   736  It is also possible to specify a value to increase or decrease by each time. You do this by adding another two dots ( .. ) and the value to step by.
   737
   738  for_loop_stepping.sh
   739  #!/bin/bash
   740  # Basic range with steps for loop
   741  for value in {10..0..2}
   742  do
   743  echo $value
   744  done
   745  echo All done
   746  ./for_loop.sh
   747  10
   748  8
   749  6
   750  4
   751  2
   752  0
   753  All done
   754  One of the more useful applications of for loops is in the processing of a set of files. To do this we may use wildcards. Let's say we want to convert a series of .html files over to .php files.
   755
   756  convert_html_to_php.sh
   757  #!/bin/bash
   758  # Make a php copy of any html files
   759  for value in $1/*.html
   760  do
   761  cp $value $1/$( basename -s .html $value ).php
   762  done
   763  Controlling Loops: Break and Continue
   764  Most of the time your loops are going to through in a smooth and ordely manner. Sometimes however we may need to intervene and alter their running slightly. There are two statements we may issue to do this.
   765
   766  Break
   767  The break statement tells Bash to leave the loop straight away. It may be that there is a normal situation that should cause the loop to end but there are also exceptional situations in which it should end as well. For instance, maybe we are copying files but if the free disk space get's below a certain level we should stop copying.
   768
   769  copy_files.sh
   770  #!/bin/bash
   771  # Make a backup set of files
   772  for value in $1/*
   773  do
   774  used=$( df $1 | tail -1 | awk '{ print $5 }' | sed 's/%//' )
   775  if [ $used -gt 90 ]
   776  then
   777  echo Low disk space 1>&2
   778  break
   779  fi
   780  cp $value $1/backup/
   781  done
   782  Continue
   783  The continue statement tells Bash to stop running through this iteration of the loop and begin the next iteration. Sometimes there are circumstances that stop us from going any further. For instance, maybe we are using the loop to process a series of files but if we happen upon a file which we don't have the read permission for we should not try to process it.
   784
   785  copy_check.sh
   786  #!/bin/bash
   787  # Make a backup set of files
   788  for value in $1/*
   789  do
   790  if [ ! -r $value ]
   791  then
   792  echo $value not readable 1>&2
   793  continue
   794  fi
   795  cp $value $1/backup/
   796  done
   797  Select
   798  The select mechanism allows you to create a simple menu system. It has the following format:
   799
   800  select var in <list>
   801  do
   802  <commands>
   803  done
   804
   805  When invoked it will take all the items in list (similar to other loops this is a space separated set of items) and present them on the screen with a number before each item. A prompt will be printed after this allowing the user to select a number. When they select a number and hit enter the corresponding item will be assigned to the variable var and the commands between do and done are run. Once finished a prompt will be displayed again so the user may select another option.
   806
   807  A few points to note:
   808
   809  No error checking is done. If the user enters something other than a number or a number not corresponding to an item then var becomes null (empty)
   810  If the user hits enter without entering any data then the list of options will be displayed again.
   811  The loop will end when an EOF signal is entered or the break statement is issued.
   812  You may change the system variable PS3 to change the prompt that is displayed.
   813  Here is a simple example to illustrate it's usage:
   814
   815  select_example.sh
   816  #!/bin/bash
   817  # A simple menu system
   818  names='Kyle Cartman Stan Quit'
   819  PS3='Select character: '
   820  select name in $names
   821  do
   822  if [ $name == 'Quit' ]
   823  then
   824  break
   825  fi
   826  echo Hello $name
   827  done
   828  echo Bye
   829  Let's break it down:
   830
   831  Line 4 - Set up a variable with the list of characters and a last option which we may select to quit. Note that the items are separated by a space.
   832  Line 6 - Change the value of the system variable PS3 so that the prompt is set to something a little more descriptive. (By default it is #?)
   833  Lines 10 - 13 - If the last option, 'Quit', is selected then break out of the select loop.
   834  Line 14 - Print out a message just to demonstrate the mechanism has worked. You may have as many commands here as you like.
   835  Line 17 - Print a message just to show that the script has continued as normal after the select loop.
   836  And now let's run the Bash script:
   837
   838  ./select_example.sh
   839  1) Kyle     3) Stan
   840  2) Cartman  4) Quit
   841  Select character: 2
   842  Hello Cartman
   843  Select Character: 1
   844  Hello Kyle
   845  Select character: 4
   846  Bye
   847  Summary
   848  while do done
   849  Perform a set of commands while a test is true.
   850  until do done
   851  Perform a set of commands until a test is true.
   852  for do done
   853  Perform a set of commands for each item in a list.
   854  break
   855  Exit the currently running loop.
   856  continue
   857  Stop this iteration of the loop and begin the next iteration.
   858  select do done
   859  Display a simple menu system for selecting items from a list.
   860  Clarity
   861  There are several Bash loop mechanisms. Pick the one which makes your code the easiest to follow.
   862  Planning
   863  Now that your scripts are getting a little more complex you will probably want to spend a little bit of time thinking about how you structure them before diving in.
   864
   865  Tue Sep  4 20:48:34 EDT 2018
   866  NOTE: dd compression on the fly
   867
   868  sudo bash -c "dd if=/dev/sda2 | gzip > /media/disk/sda2-backup-10august09.gz"
   869  sudo dd if=/dev/sda2 | bzip2 --best > /media/disk/$(date +%Y%m%d_%H%M%S)_sda2-backup.bz2
   870  lzma -z9ev
   871
   872  ---------------------------------------------
   873