]> hackdaworld.org Git - hdw-linux/hdw-linux.git/blob - scripts/Download
udev, openssh fixes
[hdw-linux/hdw-linux.git] / scripts / Download
1 #! /bin/sh
2 #
3 # author:       miguel & hackbard
4 #
5 # This is the Download script used to fetch all the hdw-linux sources
6 # wheather it is from their original locations or from a mirror site
7 #
8 # To run the script use ./scripts/Download [ options ]
9
10 mirror=""
11 package=""
12 no_cvs="0"
13 list_op=""
14 remove="0"
15
16 while [ "$1" ] ; do
17         case "$1" in
18                 -mirror)        mirror="$2"             ; shift 2 ;;
19                 -package)       package="$2"            ; shift 2 ;;
20                 -list-missing)  list_op="missing"       ; shift 1 ;;
21                 -list-broken)   list_op="broken"        ; shift 1 ;;
22                 -list-unknown)  list_op="unknown"       ; shift 1 ;;
23                 -remove-unknown)
24                                 list_op="unknown"
25                                 remove="1"              ; shift 1 ;;
26                 -check-new)     list_op="check_new"     ; shift 1 ;;
27                 -no_cvs)        no_cvs="1"              ; shift 1 ;;
28                 *)              echo 
29                                 echo "usage:"
30                                 echo 
31                                 echo " ./scripts/Download"
32                                 echo " ./scripts/Download -package <name>"
33                                 echo 
34                                 echo " use -no_cvs option to avoid refetching cvs-packages"
35                                 echo
36                                 echo "to fetch sources from a mirror:"
37                                 echo
38                                 echo " ./scripts/Download -mirror <mirror>"
39                                 echo " ./scripts/Download -package <name> -mirror <mirror>"
40                                 echo
41                                 echo "other options:"
42                                 echo
43                                 echo " ./scripts/Download -list-missing"
44                                 echo " ./scripts/Download -list-broken"
45                                 echo " ./scripts/Download -list-unknown"
46                                 echo " ./scripts/Download -check-new"
47                                 echo " ./scripts/Download -remove-unknown"
48                                 echo " ./scripts/Download -no_cvs"
49                                 echo
50                                 exit 1 ;;
51         esac
52 done
53
54 # we need to know some of the hdw specific variables.
55 . ./Config
56
57 # running subroutines
58 . ./scripts/subroutines
59
60 wget_cmd="wget --tries=3 --passive-ftp "
61 curl="curl --disable-epsv -s"
62
63 # -list-missing & -list-broken (& -list-unknown)
64 if [ ! -z "$list_op" ] ; then
65
66 for dir in `ls -A --ignore="CVS" ./download`; do
67         if [ -f ./download/$dir/PKGS ] ; then
68                 cat ./download/$dir/PKGS | while read pkg file location; do
69                 
70                 case "$list_op" in
71                         broken)
72                         server=`echo $location | awk -F: '{print $1}'`
73                         if [ "$server" = "ftp" ] ; then
74                                 # not very nice!
75                                 if $wget_cmd -O .ftp-index \
76                                 $location 2> /dev/null; then
77                                         [ -z "`grep $file .ftp-index`" ] && \
78                         echo_r "$dir $file broken ($server - package)"
79                                         rm -f .ftp-index
80                                 else
81                         echo_r "$dir $file broken ($server - directory)"
82                                 fi
83                         elif [ "$server" = "http" ] ; then
84                                 $wget_cmd --spider $location$file 2> /dev/null
85                                 [ "$?" = "1" ] && \
86                         echo_r "$dir $file broken ($server - package)"
87                         else
88                         echo "$dir $file -> unable to search on $server server"
89                         fi ;;
90
91                         check_new)
92                         if [ -z "`type -p curl`" ]; then
93                                 echo "you need curl installed to use -check-new"
94                                 exit 1
95                         fi
96                         prune=".sign|.deb|.rpm|.md5|.diff|.patch"
97                         c_v="`grep '^#\ \[V\]' \
98                                 \`find packages/*/$pkg/$pkg\` |
99                                 sed 's/^#\ \[V\]//'`"
100                         server=`echo $location | awk -F: '{print $1}'`
101                         ending="`detect_file_ending $file`"
102                         mkdir -p ./check_tmp
103                         if [ "$server" = "http" ] ; then
104                                 $curl $location | grep $pkg | grep $ending | grep -vE $prune | \
105                                         sed -e 's/imgsrc.*"\//IMG\ "/'g -e 's/<IMG .*">//'g | \
106                                         sed -e 's/ahref/A\ /'g -e 's/<\/a/<\/A/'g | \
107                                         sed -e 's/<A .*">//'g -e 's/<\/A.*//'g | \
108                                         sed 's/\ //g' > ./check_tmp/all-files
109                         elif [ "$server" = "ftp" ] ; then
110                                 $curl $location | tr -s ' ' | cut -f9 -d' ' | \
111                                         grep $pkg | grep $ending | grep -vE $prune \
112                                         > ./check_tmp/all-files
113                         else
114                                 echo "$pkg: only http/ftp servers supported"
115                                 exit 1
116                         fi
117                         if [ -f ./check_tmp/all-files ]; then
118                                 echo "$pkg:"
119                                 echo "current version: $c_v"
120                                 echo "new versions:"
121                                 cat ./check_tmp/all-files
122                                 echo
123                         fi ;;
124
125                         missing)
126                 if [ "`echo $location | awk -F: '{ print $1 }'`" = "cvs" ]; then
127                         if [ "$no_cvs" = "0" ] ; then
128                                 echo -n "$pkg is cvs-package, "
129                                 if [ ! -f ./download/$dir/$pkg/$file ] ; then
130                                         echo_r "missing"
131                                 else
132                                         echo_g "available"
133                                 fi
134                         fi
135                 else
136                         if [ ! -f ./download/$dir/$pkg/$file ] ; then
137                                 echo -n "$pkg "
138                                 echo_r "missing"
139                         fi
140                 fi ;;
141
142                         unknown)
143                         if [ -d ./download/$dir/$pkg ] ; then
144                         objects="`ls -A ./download/$dir/$pkg`"
145                         for i in $objects; do
146                                 known_f="`grep ^$pkg ./download/$dir/PKGS | \
147                                 awk '{ print $2 }'`"
148                                 if [ -z "`echo $known_f | grep $i`" ] ; then
149                                         echo "./download/$dir/$pkg/$i unknown"
150                                         if [ "$remove" = "1" ] ; then
151                                                 echo "removing ..."
152                                                 rm ./download/$dir/$pkg/$i
153                                         fi
154                                 fi
155                         done
156                         fi ;;
157                 esac
158
159                 done
160         fi
161 done
162
163 exit 0
164 fi
165
166 # download a package
167 if [ ! -z "$package" ] ; then
168
169 for dir in `ls -A --ignore="CVS" ./download`; do
170         if [ -f ./download/$dir/PKGS ] ; then
171                 cat ./download/$dir/PKGS | while read pkg file location; do
172                 
173                 if [ "$package" = "$pkg" ] ; then
174                 
175                 is_cvs="`echo $location | awk -F: '{ print $1 }'`"      
176
177                 # cvs
178                 if [ "${is_cvs}" = "cvs" -o "${is_cvs}" = "sshcvs" ] ; then
179                         if [ "$no_cvs" = "0" ] ; then
180
181                         user=`echo $location | awk -F: '{ print $2 }'`
182                         loc=`echo $location | awk -F: '{ print $3 }'`
183                         repos=`echo $location | awk -F: '{ print $4 }'`
184                         module=`echo $location | awk -F: '{ print $5 }'`
185                         #passwd=`echo $location | awk -F: '{ print $6 }'`
186                         
187                         echo "downloading $pkg using cvs ..."
188                         [ ! -f $HOME/.cvspass ] && touch $HOME/.cvspass
189                         mkdir -p ./download/$dir/$pkg ; cd ./download/$dir/$pkg
190                         rm -rf $file $module
191                         dlerror=0
192                         if [ "${is_cvs}" = "cvs" ] ; then
193                                 cvs -z 9 -d:pserver:${user}@${loc}:/${repos} co $module
194                                 [ "$?" != "0" ] && dlerror=1
195                         else
196                                 export CVS_RSH=ssh
197                                 cvs -d ${user}@${loc}:/${repos} co $module
198                                 [ "$?" != "0" ] && dlerror=1
199                                 unset CVS_RSH
200                         fi
201                         if [ "$dlerror" = "0" ] ; then
202                                 echo
203                                 echo "making package ..."
204                                 tar cf `echo $file | sed 's/.tar.bz2//'`.tar \
205                                         $module/
206                                 bzip2 `echo $file | sed 's/.tar.bz2//'`.tar
207                         fi
208                         rm -rf $module
209                         cd ../../..
210                         
211                         fi
212                 # tar.bz2               
213                 elif [ "${is_cvs}" != "cvs" ] ; then
214                         [ ! -z "$mirror" ] && \
215                         location="${mirror}/hdw-linux-${hdw_version}/$dir/$pkg/"
216                 echo "Downloading $dir - package $file ..."
217                 echo
218                 # possibly the package exists
219                 mkdir -p ./download/$dir/$pkg ; cd ./download/$dir/$pkg
220                                 tolf="`find ../.. -name $file | tail -1`"
221                                 if [ -n "$tolf" ] ; then
222                                         [ -d $tolf ] && tolf=""
223                                 fi
224                                 if [ -f ./$file ] ; then
225                                         echo "$file allready downloaded."
226                                         echo
227                                 elif [ -n "$tolf" ] ; then
228                                         ln -svf $tolf .
229                                 else
230                                 
231                                         $wget_cmd -c $location$file \
232                                                 -O pkg-src.in &&
233                                         mv pkg-src.in $file
234                                         echo "done."
235                                 fi
236                 cd ../../..
237                 fi
238
239                 fi
240                 done
241         fi
242 done
243 fi
244
245 # download all packages needed for defined target
246 if [ -z "$package" ] ; then
247
248 d_append=""
249 [ "$no_cvs" = "1" ] && d_append="-no_cvs"
250
251 for pkg in `./scripts/Download -list-missing | awk '{ print $1 }'`; do
252         if [ ! -z "$mirror" ] ; then 
253                 cmd_mirror="-mirror $mirror"
254         else
255                 cmd_mirror=""
256         fi
257         echo
258         . ./scripts/Download -package $pkg $cmd_mirror $d_append
259         echo
260 done
261
262 fi
263
264 # show list_unknown stuff
265 # coming soon :)