scripts/Download

   1 #! /bin/sh
   2 #
   3 # author:       miguel & hackbard
   4 #
   5 # This is the Download script used to fetch all the hdw-linux sources
   6 # wheather it is from their original locations or from a mirror site
   7 #
   8 # To run the script use ./scripts/Download [ options ]
   9
  10 mirror=""
  11 package=""
  12 no_cvs="0"
  13 list_op=""
  14 remove="0"
  15
  16 while [ "$1" ] ; do
  17         case "$1" in
  18                 -mirror)        mirror="$2"             ; shift 2 ;;
  19                 -package)       package="$2"            ; shift 2 ;;
  20                 -list-missing)  list_op="missing"       ; shift 1 ;;
  21                 -list-broken)   list_op="broken"        ; shift 1 ;;
  22                 -list-unknown)  list_op="unknown"       ; shift 1 ;;
  23                 -remove-unknown)
  24                                 list_op="unknown"
  25                                 remove="1"              ; shift 1 ;;
  26                 -check-new)     list_op="check_new"     ; shift 1 ;;
  27                 -no_cvs)        no_cvs="1"              ; shift 1 ;;
  28                 *)              echo
  29                                 echo "usage:"
  30                                 echo
  31                                 echo " ./scripts/Download"
  32                                 echo " ./scripts/Download -package <name>"
  33                                 echo
  34                                 echo " use -no_cvs option to avoid refetching cvs-packages"
  35                                 echo
  36                                 echo "to fetch sources from a mirror:"
  37                                 echo
  38                                 echo " ./scripts/Download -mirror <mirror>"
  39                                 echo " ./scripts/Download -package <name> -mirror <mirror>"
  40                                 echo
  41                                 echo "other options:"
  42                                 echo
  43                                 echo " ./scripts/Download -list-missing"
  44                                 echo " ./scripts/Download -list-broken"
  45                                 echo " ./scripts/Download -list-unknown"
  46                                 echo " ./scripts/Download -check-new"
  47                                 echo " ./scripts/Download -remove-unknown"
  48                                 echo " ./scripts/Download -no_cvs"
  49                                 echo
  50                                 exit 1 ;;
  51         esac
  52 done
  53
  54 # we need to know some of the hdw specific variables.
  55 . ./Config
  56
  57 # running subroutines
  58 . ./scripts/subroutines
  59
  60 wget_cmd="wget --tries=3 --passive-ftp "
  61 curl="curl --disable-epsv -s"
  62
  63 # -list-missing & -list-broken (& -list-unknown)
  64 if [ ! -z "$list_op" ] ; then
  65
  66 for dir in `ls -A --ignore="CVS" ./download`; do
  67         if [ -f ./download/$dir/PKGS ] ; then
  68                 cat ./download/$dir/PKGS | while read pkg file location; do
  69
  70                 case "$list_op" in
  71                         broken)
  72                         server=`echo $location | awk -F: '{print $1}'`
  73                         if [ "$server" = "ftp" ] ; then
  74                                 # not very nice!
  75                                 if $wget_cmd -O .ftp-index \
  76                                 $location 2> /dev/null; then
  77                                         [ -z "`grep $file .ftp-index`" ] && \
  78                         echo_r "$dir $file broken ($server - package)"
  79                                         rm -f .ftp-index
  80                                 else
  81                         echo_r "$dir $file broken ($server - directory)"
  82                                 fi
  83                         elif [ "$server" = "http" ] ; then
  84                                 $wget_cmd --spider $location$file 2> /dev/null
  85                                 [ "$?" = "1" ] && \
  86                         echo_r "$dir $file broken ($server - package)"
  87                         else
  88                         echo "$dir $file -> unable to search on $server server"
  89                         fi ;;
  90
  91                         check_new)
  92                         if [ -z "`type -p curl`" ]; then
  93                                 echo "you need curl installed to use -check-new"
  94                                 exit 1
  95                         fi
  96                         prune=".sign|.deb|.rpm|.md5|.diff|.patch"
  97                         c_v="`grep '^#\ \[V\]' \
  98                                 \`find packages/*/$pkg/$pkg\` |
  99                                 sed 's/^#\ \[V\]//'`"
 100                         server=`echo $location | awk -F: '{print $1}'`
 101                         ending="`detect_file_ending $file`"
 102                         mkdir -p ./check_tmp
 103                         if [ "$server" = "http" ] ; then
 104                                 $curl $location | grep $pkg | grep $ending | grep -vE $prune | \
 105                                         sed -e 's/imgsrc.*"\//IMG\ "/'g -e 's/<IMG .*">//'g | \
 106                                         sed -e 's/ahref/A\ /'g -e 's/<\/a/<\/A/'g | \
 107                                         sed -e 's/<A .*">//'g -e 's/<\/A.*//'g | \
 108                                         sed 's/\ //g' > ./check_tmp/all-files
 109                         elif [ "$server" = "ftp" ] ; then
 110                                 $curl $location | tr -s ' ' | cut -f9 -d' ' | \
 111                                         grep $pkg | grep $ending | grep -vE $prune \
 112                                         > ./check_tmp/all-files
 113                         else
 114                                 echo "$pkg: only http/ftp servers supported"
 115                                 exit 1
 116                         fi
 117                         if [ -f ./check_tmp/all-files ]; then
 118                                 echo "$pkg:"
 119                                 echo "current version: $c_v"
 120                                 echo "new versions:"
 121                                 cat ./check_tmp/all-files
 122                                 echo
 123                         fi ;;
 124
 125                         missing)
 126                 if [ "`echo $location | awk -F: '{ print $1 }'`" = "cvs" ]; then
 127                         if [ "$no_cvs" = "0" ] ; then
 128                                 echo -n "$pkg is cvs-package, "
 129                                 if [ ! -f ./download/$dir/$pkg/$file ] ; then
 130                                         echo_r "missing"
 131                                 else
 132                                         echo_g "available"
 133                                 fi
 134                         fi
 135                 else
 136                         if [ ! -f ./download/$dir/$pkg/$file ] ; then
 137                                 echo -n "$pkg "
 138                                 echo_r "missing"
 139                         fi
 140                 fi ;;
 141
 142                         unknown)
 143                         if [ -d ./download/$dir/$pkg ] ; then
 144                         objects="`ls -A ./download/$dir/$pkg`"
 145                         for i in $objects; do
 146                                 known_f="`grep ^$pkg ./download/$dir/PKGS | \
 147                                 awk '{ print $2 }'`"
 148                                 if [ -z "`echo $known_f | grep $i`" ] ; then
 149                                         echo "./download/$dir/$pkg/$i unknown"
 150                                         if [ "$remove" = "1" ] ; then
 151                                                 echo "removing ..."
 152                                                 rm ./download/$dir/$pkg/$i
 153                                         fi
 154                                 fi
 155                         done
 156                         fi ;;
 157                 esac
 158
 159                 done
 160         fi
 161 done
 162
 163 exit 0
 164 fi
 165
 166 # download a package
 167 if [ ! -z "$package" ] ; then
 168
 169 for dir in `ls -A --ignore="CVS" ./download`; do
 170         if [ -f ./download/$dir/PKGS ] ; then
 171                 cat ./download/$dir/PKGS | while read pkg file location; do
 172
 173                 if [ "$package" = "$pkg" ] ; then
 174
 175                 is_cvs="`echo $location | awk -F: '{ print $1 }'`"
 176
 177                 # cvs
 178                 if [ "${is_cvs}" = "cvs" -o "${is_cvs}" = "sshcvs" ] ; then
 179                         if [ "$no_cvs" = "0" ] ; then
 180
 181                         user=`echo $location | awk -F: '{ print $2 }'`
 182                         loc=`echo $location | awk -F: '{ print $3 }'`
 183                         repos=`echo $location | awk -F: '{ print $4 }'`
 184                         module=`echo $location | awk -F: '{ print $5 }'`
 185                         #passwd=`echo $location | awk -F: '{ print $6 }'`
 186
 187                         echo "downloading $pkg using cvs ..."
 188                         [ ! -f $HOME/.cvspass ] && touch $HOME/.cvspass
 189                         mkdir -p ./download/$dir/$pkg ; cd ./download/$dir/$pkg
 190                         rm -rf $file $module
 191                         dlerror=0
 192                         if [ "${is_cvs}" = "cvs" ] ; then
 193                                 cvs -z 9 -d:pserver:${user}@${loc}:/${repos} co $module
 194                                 [ "$?" != "0" ] && dlerror=1
 195                         else
 196                                 export CVS_RSH=ssh
 197                                 cvs -d ${user}@${loc}:/${repos} co $module
 198                                 [ "$?" != "0" ] && dlerror=1
 199                                 unset CVS_RSH
 200                         fi
 201                         if [ "$dlerror" = "0" ] ; then
 202                                 echo
 203                                 echo "making package ..."
 204                                 tar cf `echo $file | sed 's/.tar.bz2//'`.tar \
 205                                         $module/
 206                                 bzip2 `echo $file | sed 's/.tar.bz2//'`.tar
 207                         fi
 208                         rm -rf $module
 209                         cd ../../..
 210
 211                         fi
 212                 # tar.bz2
 213                 elif [ "${is_cvs}" != "cvs" ] ; then
 214                         [ ! -z "$mirror" ] && \
 215                         location="${mirror}/hdw-linux-${hdw_version}/$dir/$pkg/"
 216                 echo "Downloading $dir - package $file ..."
 217                 echo
 218                 # possibly the package exists
 219                 mkdir -p ./download/$dir/$pkg ; cd ./download/$dir/$pkg
 220                                 tolf="`find ../.. -name $file | tail -1`"
 221                                 if [ -f ./$file ] ; then
 222                                         echo "$file allready downloaded."
 223                                         echo
 224                                 elif [ -n "$tolf" ] ; then
 225                                         ln -svf $tolf .
 226                                 else
 227
 228                                         $wget_cmd -c $location$file \
 229                                                 -O pkg-src.in &&
 230                                         mv pkg-src.in $file
 231                                         echo "done."
 232                                 fi
 233                 cd ../../..
 234                 fi
 235
 236                 fi
 237                 done
 238         fi
 239 done
 240 fi
 241
 242 # download all packages needed for defined target
 243 if [ -z "$package" ] ; then
 244
 245 d_append=""
 246 [ "$no_cvs" = "1" ] && d_append="-no_cvs"
 247
 248 for pkg in `./scripts/Download -list-missing | awk '{ print $1 }'`; do
 249         if [ ! -z "$mirror" ] ; then
 250                 cmd_mirror="-mirror $mirror"
 251         else
 252                 cmd_mirror=""
 253         fi
 254         echo
 255         . ./scripts/Download -package $pkg $cmd_mirror $d_append
 256         echo
 257 done
 258
 259 fi
 260
 261 # show list_unknown stuff
 262 # coming soon :)