/var/lib/sorcery/modules/url_handlers/url_http
1 #!/bin/bash
2 #---------------------------------------------------------------------
3 ##
4 ##=head1 SYNOPSIS
5 ##
6 ## Url handler functions for downloading http, https, and ftp urls
7 ##
8 ##=head1 DESCRIPTION
9 ##
10 ## This file contains functions for downloading and verifying
11 ## http, https, and ftp urls. This file uses the "wget" program.
12 ##
13 ##=head1 COPYRIGHT
14 ##
15 ## Copyright 2002 by the Source Mage Team
16 ##
17 ##=head1 FUNCTIONS
18 ##
19 ##=over 4
20 ##
21 #---------------------------------------------------------------------
22
23 #---------------------------------------------------------------------
24 ##=item url_http_bucketize <url>
25 ##
26 ## echoes the download handler - wget
27 ##
28 #---------------------------------------------------------------------
29 function url_http_bucketize() {
30 echo $HTTP_DL_HANDLER
31 }
32 function url_ftp_bucketize() {
33 echo $HTTP_DL_HANDLER
34 }
35 function url_https_bucketize() {
36 echo $HTTP_DL_HANDLER
37 }
38
39 #---------------------------------------------------------------------
40 ##=item url_http_verify <url>
41 ##
42 ## Verifies the specified http url. Returns true if the url exists
43 ## OR if the url is an empty string.
44 ##
45 #---------------------------------------------------------------------
46 function url_http_verify() {
47 local URL=$1
48 local hints="$2"
49 local check_certificate
50
51 if list_find "$hints" no-check-certificate; then
52 check_certificate=--no-check-certificate
53 fi
54
55 if [ -n "$URL" ]; then
56 if OUTPUT=`wget --passive-ftp -t 1 -T 30 --spider $check_certificate "$URL" 2>&1`; then
57 true
58 else
59 echo $OUTPUT
60 false
61 fi
62 fi
63 }
64
65
66 #---------------------------------------------------------------------
67 ##=item url_https_verify <url>
68 ##
69 ## Verifies the specified https url. Returns true if the url exists
70 ## OR if the url is an empty string.
71 ##
72 #---------------------------------------------------------------------
73 function url_https_verify() {
74 url_http_verify "$@"
75 }
76
77
78 #---------------------------------------------------------------------
79 ##=item url_ftp_verify <url>
80 ##
81 ## Verifies the specified ftp url. Echos results of wget if file
82 ## is not found.
83 ##
84 ##
85 ## @Implementation_note wget --spider still downloads ftp files in full rather
86 ## @Implementation_note than just checking that the file is there. To get
87 ## @Implementation_note around this problem, we download the directory and see
88 ## @Implementation_note if the file is in the directory listing.
89 ##
90 #---------------------------------------------------------------------
91 function url_ftp_verify() {
92 local URL=$1
93
94 if [ -n "$URL" ]; then
95 local FILENAME
96 local DIRECTORY
97 smgl_basename $URL FILENAME
98 smgl_dirname $URL DIRECTORY
99 local OUTPUT=`wget --passive-ftp -t 1 -T 30 -O - --spider -S "$DIRECTORY/" 2>&1`
100
101 if real_list_find "$OUTPUT" "$FILENAME"; then
102 rm -f .listing
103 else
104 echo $OUTPUT | sed 's/LIST.*//g'
105 [ -f .listing ] && cat .listing
106 rm -f .listing
107 false
108 fi
109 fi
110 }
111
112 #---------------------------------------------------------------------
113 ##=item url_<prefix>_hostname <url>
114 ##
115 ## Gets the hostname out of the url
116 #---------------------------------------------------------------------
117 function url_http_hostname() {
118 echo $1|sed 's:^.*//\([^/]*\).*$:\1:'
119 }
120 function url_https_hostname() {
121 url_http_hostname $1
122 }
123 function url_ftp_hostname() {
124 url_http_hostname $1
125 }
126
127 #---------------------------------------------------------------------
128 ##=item url_<prefix>_netselect <url>
129 ##
130 ## Run netselect on the url, netselect understands these urls and
131 ## we can take advantage of its multi-A record handling
132 ##
133 #---------------------------------------------------------------------
134 function url_http_netselect() {
135 netselect -t 3 -s 1000 "$@" 2>/dev/null
136 }
137 function url_https_netselect() {
138 netselect -t 3 -s 1000 "$@" 2>/dev/null
139 }
140 function url_ftp_netselect() {
141 netselect -t 3 -s 1000 "$@" 2>/dev/null
142 }
143
144 #-------------------------------------------------------------------------
145 ## expand a url to ALL of the mirrors we think may be related
146 ## Take the url, find its hostname, if a file in $MIRRORS matches
147 ## generate new urls from all the mirrors in the file, the original url
148 ## is kept at the top of the list and not duplicated
149 ##
150 ## @param a list of urls
151 ## @stdout the expanded form of those urls
152 #-------------------------------------------------------------------------
153 function url_http_expand() {
154 local A=$'\a'
155 local URL rep tgt my_hostname each
156 # put the requested url first
157 echo "$@"
158 for URL in $@ ; do
159 my_hostname=$(url_hostname ${URL})
160 # a mirror is listed in the mirror listings as either
161 # ftp://foo.somemirror.org/stuff
162 # OR
163 # ftp://foo.somemirror.org
164 # note neither ends in a '/' but one has a '/' at the end of the
165 # hostname, the other has an end of line. The \(/\|\$\) stuff matches
166 # either
167 for each in $(grep -lr "://$my_hostname\(/\|\$\)" --exclude-dir=config $MIRRORS); do
168 rep=$(grep "://$my_hostname\(/\|\$\)" $each|awk '{print $NF; exit 0; }')
169 for tgt in $(awk '{print $NF}' $each|grep -v '^Custom$'); do
170 echo ${URL}|sed "s$A${rep}$A${tgt}$A"
171 done
172 done |grep -v $my_hostname
173 done
174 }
175
176 function url_ftp_expand() {
177 url_http_expand "$@"
178 }
179 # note that there is no https_expand
180
181 #---------------------------------------------------------------------
182 ##=back
183 ##
184 ##=head1 LICENSE
185 ##
186 ## This software is free software; you can redistribute it and/or modify
187 ## it under the terms of the GNU General Public License as published by
188 ## the Free Software Foundation; either version 2 of the License, or
189 ## (at your option) any later version.
190 ##
191 ## This software is distributed in the hope that it will be useful,
192 ## but WITHOUT ANY WARRANTY; without even the implied warranty of
193 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
194 ## GNU General Public License for more details.
195 ##
196 ## You should have received a copy of the GNU General Public License
197 ## along with this software; if not, write to the Free Software
198 ## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
199 ##
200 #---------------------------------------------------------------------