/var/lib/sorcery/modules/libsummon
1 #!/bin/bash
2 #---------------------------------------------------------------------
3 ## @Synopsis Internal api for summoning/printing spell file information. Mainly for use by cast and summon, who both need to "summon".
4 ##
5 ## This just wraps up calls to run_details, get_spell_files_and_urls
6 ## and download_files. Along with doing the logging and other stuff.
7 ## This is mainly to reduce duplication of code between cast and summon
8 ## and to make it so cast doesn't have to actually run the summon script.
9 ##
10 ## @Copyright Copyright (C) 2004 The Source Mage Team <http://www.sourcemage.org>
11 ## @Copyright Copyright (C) 2005 The Source Mage Team <http://www.sourcemage.org>
12 ##
13 ## @Contributors Andrew Stitt <astitt@sourcemage.org)
14 ## @Contributors Paul Mahon <pmahon@sourcemage.org)
15 #---------------------------------------------------------------------
16
17 #---------------------------------------------------------------------
18 ## @param Spell name to summon
19 ## @return 0 if all the source urls were downloaded/found
20 ## 1 if any of them failed.
21 ## As the name would imply, summon a spell given its name.
22 ## The spell is sourced in a subshell and thus no variable leakage
23 ## will occur to the caller (although the function could be effected
24 ## by leakage from other places). This function will take care of
25 ## locking out other processes from downloading the same spell.
26 ## Also takes care of making entries in the activity log
27 ##
28 #---------------------------------------------------------------------
29 function summon_spell() {
30 $STD_DEBUG
31 local SPELL=$1
32 local rc=1
33 if [ -n "${SPELL}" ] && lock_resources "summon" "${SPELL}"; then
34 local dl_dir=$TMP_DIR/${SPELL:-none}
35 debug "libsummon" "dl_dir is $dl_dir"
36 mkdir -p $dl_dir && pushd $dl_dir &>/dev/null &&
37
38 # run in a subshell rather than try to unset_details
39 # and instead jump through hoops to get the return code correctly
40 ( local rc=1
41 run_details &&
42 run_spell_file DOWNLOAD download
43 rc=$?
44
45 if [[ $rc == 0 ]]; then
46 activity_log "summon" "$SPELL" "$VERSION" "success"
47 else
48 activity_log "summon" "$SPELL" "$VERSION" "failure" "download failed"
49 fi
50
51 # this actually only returns from the subshell, not the function
52 return $rc
53 ) && popd &>/dev/null
54 rc=$?
55 rm -rf $dl_dir
56 unlock_resources "summon" "${SPELL}"
57 fi
58 return $rc
59 }
60
61 #-------------------------------------------------------------------------
62 ## Helper routine to dump out the files and urls for a spell given that
63 ## the caller has run run_details.
64 ##
65 ## @Stdout multiple lines, one per SOURCE/SOURCEx as "SOURCE SOURCE_URL[0] SOURCE_URL[1]"
66 #-------------------------------------------------------------------------
67 function get_spell_files_and_urls() {
68 $STD_DEBUG
69 local url src
70 for src in $(real_get_source_nums SOURCE); do
71 url="$src"'_URL[*]'
72 src="${!src}"
73 [[ ${src} ]] && echo ${src} ${!url}
74 done
75 }
76
77
78 #-------------------------------------------------------------------------
79 ## @param none
80 ##
81 ## Call acquire_src for each SOURCEx_URL
82 ##
83 #-------------------------------------------------------------------------
84 function real_default_sorcery_download() {
85 $STD_DEBUG
86 local SOURCE_NUMBER
87 for SOURCE_NUMBER in $(real_get_source_nums X); do
88 SOURCE_NUMBER=${SOURCE_NUMBER/X/}
89 acquire_src $SOURCE_NUMBER
90 done
91 }
92
93
94 #-------------------------------------------------------------------------
95 ## @param source number or '' for the first $SOURCE
96 ##
97 ## Acquire the source. Check locally unless options insist on not
98 ## doing so.
99 ##
100 #-------------------------------------------------------------------------
101 function real_acquire_src() {
102 $STD_DEBUG
103 local DLNUM="$1"
104 local SVAR="SOURCE${DLNUM}"
105 local target=${!SVAR}
106
107 # maybe we already have the file
108 # but we can only use it if not FORCE_DOWNLOADing
109 is_downloaded "$target" "$DLNUM" \
110 "for spell ${SPELL_COLOR}${SPELL}${DEFAULT_COLOR} " && return 0
111
112 # else the source must be downloaded/updated
113 download_src "$DLNUM"
114 }
115
116 #-------------------------------------------------------------------------
117 ## @param download target
118 ## @param source number
119 ## @return 0 if the target already exsists and FORCE_DOWNLOADing is disabled
120 ## @return 1 otherwise
121 ##
122 ## Checks if the download target is already downloaded,
123 ## Inspects $FORCE_DOWNLOAD and ${FORCE_DOWNLOAD[$srcnum]}
124 ## if either are set then downloading is forced, a local copy of the
125 ## file is ignored.
126 ##
127 #-------------------------------------------------------------------------
128 function is_downloaded() {
129 local target="$1"
130 local dlnum="${2:-1}"
131 local reason="$3"
132
133 if file_exists "$SOURCE_CACHE/$target " &&
134 ! [[ ${FORCE_DOWNLOAD} ]] &&
135 ! [[ ${FORCE_DOWNLOAD[$dlnum]} ]] ; then
136 # file_exists does a fuzzy match with gz, tgz and bz2
137 # lookup what file it actually found and use that for the message
138 local real_target=$(guess_filename $SOURCE_CACHE/$target)
139 message "${MESSAGE_COLOR}Found source file" \
140 "${FILE_COLOR}${real_target}${DEFAULT_COLOR}" \
141 "${reason}in ${FILE_COLOR}${SOURCE_CACHE}${DEFAULT_COLOR}"
142 return 0
143 fi
144 return 1
145 }
146
147 #-------------------------------------------------------------------------
148 ## @param source number or '' for the first $SOURCE
149 ##
150 ## Expands $SOURCEx, $SOURCEx_URL[*] and $SOURCEx_HINTS
151 ## then calls download_src_args with them.
152 ##
153 #-------------------------------------------------------------------------
154 function real_download_src() {
155 $STD_DEBUG
156 local DLNUM="$1"
157 local SVAR="SOURCE${DLNUM}"
158 local SURLVAR=${SVAR}'_URL[*]'
159 local SHINTVAR=${SVAR}'_HINTS'
160
161 local target=${!SVAR}
162 local urls=${!SURLVAR}
163 local hints=${!SHINTVAR}
164
165 [[ "$target" ]] || {
166 message "Empty value in $SVAR!!"
167 return 1
168 }
169 [[ "$urls" ]] || {
170 message "No urls in $SURLVAR!!"
171 return 1
172 }
173 download_src_args "$target" "$urls" "$hints"
174 }
175
176 #-------------------------------------------------------------------------
177 ## @param file
178 ## @param url list
179 ## @param url options
180 ##
181 ## Download the resource, handles file and tree discrepencies, if a tree is
182 ## downloaded it is repackaged as the specified file for later use.
183 ##
184 ## Only tar.bz2 files are acceptable caches for repackaged trees
185 ##
186 #-------------------------------------------------------------------------
187 function download_src_args() {
188 $STD_DEBUG
189 local target="$1"
190 local urls="$2"
191 local hints="$3"
192
193 ensure_dir "$SOURCE_CACHE"
194
195 message "${MESSAGE_COLOR}Downloading source file" \
196 "${FILE_COLOR}${target}${DEFAULT_COLOR}" \
197 "${REASON}"
198
199 # get the update tree if we think there is one
200 local new_target=$target
201 unpack_for_update "$target" "$urls" "$hints" new_target guess_type
202
203 # this does the actual downloading:
204 local summon_type summon_target
205 if ! download_src_sub "$new_target" "$urls" "$hints" "$guess_type" \
206 summon_target summon_type; then
207 if [[ $guess_type == tree ]] && test -f $SOURCE_CACHE/$target; then
208 if [[ $STRICT_SCM_UPDATE == off ]] ; then
209 message "${PROBLEM_COLOR}Update of" \
210 "${FILE_COLOR}${target}${DEFAULT_COLOR}" \
211 "${PROBLEM_COLOR}failed, falling back to old" \
212 "version${DEFAULT_COLOR}"
213 # cleanup unpacked tree
214 rm -rf $new_target
215 return 0
216 fi
217 fi
218 message "${PROBLEM_COLOR}Download of" \
219 "${FILE_COLOR}${target}${DEFAULT_COLOR}" \
220 "${PROBLEM_COLOR}failed${DEFAULT_COLOR}"
221 return 1
222 fi
223
224 # look at what we got
225 if ! [[ $summon_target ]] ; then
226 message "${PROBLEM_COLOR}Empty value for downloaded target, file a bug" \
227 "if you see this.${DEFAULT_COLOR}"
228 return 255
229 fi
230
231 # repackage if necessary and move to $SOURCE_CACHE
232 if [[ "$summon_type" == tree ]] ; then
233
234 # incorrect guess, set new_target appropriatly
235 if [[ "$guess_type" == file ]] ; then
236 new_target="${target/.tar.bz2/}"
237 fi
238
239 if ! test -d $summon_target; then
240 message "${PROBLEM_COLOR}Value for downloaded target ($summon_target)" \
241 "is not a directory, but a tree was downloaded, file a bug" \
242 "if you see this.${DEFAULT_COLOR}"
243 return 255
244 fi
245 if [[ $summon_target != $new_target ]] ; then
246 mv -f $summon_target $new_target
247 fi &&
248 message "${MESSAGE_COLOR}Repackaging ${SPELL_COLOR}$target${DEFAULT_COLOR}"
249 tar --remove-files -cjf $target $new_target &&
250 rm -rf $new_target
251 elif [[ "$summon_type" == file ]] ; then
252 # there is no penalty for guessing tree when the result is a file
253 if [[ $summon_target != $target ]] ; then
254 mv -f "$summon_target" "$target"
255 fi
256 else
257 message "${PROBLEM_COLOR}Unknown download type: \"$summon_type\""\
258 "at \"$summon_target\". Please file a bug if you see this." \
259 "${DEFAULT_COLOR}"
260 return 1
261 fi
262 rm -rf $SOURCE_CACHE/$target &&
263 mv $target $SOURCE_CACHE
264 }
265
266 #-------------------------------------------------------------------------
267 ##
268 ## @param file
269 ## @param url list
270 ## @param url options
271 ## @param summon target return value, pass by reference
272 ## @param summon type return value, pass by reference
273 ##
274 ## Download the resource, first try a leapforward url, then the given
275 ## urls and finally the fallbacks.
276 #-------------------------------------------------------------------------
277 function download_src_sub() {
278
279 $STD_DEBUG
280
281 local target="$1"
282 local url_list="$2"
283 local hints="$3"
284 local guess_type=$4
285 local _summon_target=$5
286 local _summon_type=$6
287
288 {
289 [[ $guess_type == file ]] &&
290 download_from_leapforward "$target" "$url_list" "$hints" \
291 "$_summon_target" "$_summon_type" &&
292 source_sanity "$target"
293 } ||
294
295 {
296 url_download_expand_sort "$target" "$url_list" "$hints" \
297 "$_summon_target" "$_summon_type" &&
298 if [[ $guess_type == file ]]; then
299 source_sanity "$target"
300 fi
301 } ||
302
303 { # dont use fallback if the type is not a file (bug 9847)
304 [[ $guess_type == file ]] &&
305 download_from_fallback "$target" "$url_list" "$hints" \
306 "$_summon_target" "$_summon_type" &&
307 source_sanity "$target"
308 }
309
310 }
311
312 #-------------------------------------------------------------------------
313 ##
314 ## Download the specified resource from the leapforward url if one is
315 ## given.
316 ##
317 ## @param file
318 ## @param url list
319 ## @param url options
320 ## @param summon target return value, pass by reference
321 ## @param summon type return value, pass by reference
322 #-------------------------------------------------------------------------
323 function download_from_leapforward() {
324 $STD_DEBUG
325 if ! [ -n "$LEAPFORWARD_URL" ]; then
326 return 1
327 fi
328
329 local target="$1"
330 local url_list="$2"
331 local hints="$3"
332 local _summon_target=$4
333 local _summon_type=$5
334
335 message "${MESSAGE_COLOR}Attempting to get file from" \
336 "leap-forward mirror ${DEFAULT_COLOR}${LEAPFORWARD_URL}"
337 # leap forwards dont need to be expanded or sorted
338 url_download "$target" "$LEAPFORWARD_URL/$target" "$hints" \
339 "$_summon_target" "$_summon_type"
340 }
341
342 #-------------------------------------------------------------------------
343 ##
344 ## Attempt to get the resource from the fallback urls, try each one in
345 ## a random order.
346 ##
347 ## @param file
348 ## @param url list
349 ## @param url options
350 ## @param summon target return value, pass by reference
351 ## @param summon type return value, pass by reference
352 ##
353 #-------------------------------------------------------------------------
354 function download_from_fallback() {
355 $STD_DEBUG
356 local target="$1"
357 local url_list="$2"
358 local hints="$3"
359 local _summon_target=$4
360 local _summon_type=$5
361
362 message "${MESSAGE_COLOR}Attempting to get file from fall-back mirrors" \
363 "${DEFAULT_COLOR}"
364
365 # slow and painful buildup of all fallback mirrors in quasi random order
366 local i idx
367 local FALL_BACKS
368 local offset=$[${RANDOM} % $FURLNUM]
369 for (( i=0; $i < $FURLNUM; i++ )); do
370 idx=$[($i + $offset) % $FURLNUM]
371 FALL_BACKS="$FALL_BACKS ${FALLBACK_URL_MIRROR[$idx]}/$target"
372 done
373 [ -n "$FALL_BACKS" ] &&
374 # dont order fall backs, above we use a random ordering
375 url_download "$target" "$FALL_BACKS" "$hints" "$_summon_target" \
376 "$_summon_type"
377 }
378
379 #-------------------------------------------------------------------------
380 ##
381 ## Make a guess as to whether or not the resource being downloaded
382 ## will be a file or a tree if it is a tree and the cached source exists
383 ## then unpack it in the current directory so it may be updated
384 ##
385 ## @param target
386 ## @param url_list
387 ## @param hints
388 ## @param new target return value pass by reference
389 ## @param guessed type return value pass by reference
390 ##
391 ##
392 #-------------------------------------------------------------------------
393 function unpack_for_update() {
394 $STD_DEBUG
395 local target="$1"
396 local url_list="$2"
397 local hints="$3"
398 local new_target_ref=$4
399 local guess_type_ref=$5
400 local _new_target
401 local _guess_type
402
403 # hard-coded list of url prefixes that generally download trees
404 local tree_prefixes="cvs dir rsync smgl_tla svn svn_http svn_ssh git"
405 tree_prefixes="$tree_prefixes svn_https git_http hg_http bzr"
406 local prefix=$(url_get_prefix $url_list)
407 if ! list_find "$hints" file &&
408 list_find "$hints" tree || list_find "$tree_prefixes" "$prefix" ; then
409 _new_target="${target/.tar.bz2/}"
410 eval "$guess_type_ref=\"tree\""
411 eval "$new_target_ref=\"\$_new_target\""
412
413 # if $target exists in $SOURCE_CACHE and we think the download might be
414 # a tree, unpack it so the tree can be updated
415 if test -f $SOURCE_CACHE/$target && ! list_find "$hints" no_update; then
416 unpack_file_simple $target || return 1
417 if ! test -d "$_new_target" ; then
418 debug "libsummon" "unpacked $target but to somewhere strange"
419 fi
420 fi
421 else
422 eval "$new_target_ref=\"\$target\""
423 eval "$guess_type_ref=\"file\""
424 fi
425 return 0
426 }
427
428 #---------------------------------------------------------------------
429 ## Perform rudimentary source sanity checks on summoned files.
430 ## Currently only checks that files are not zero sized or html 404 notices.
431 ## Some sites, notably sourceforge, break http by having missing files
432 ## do a 30X redirect to a 404 html file which when downloaded has a
433 ## 200 (OK) status. Other sites do 30X redirect to the new location of
434 ## the requested file, which then successfully downloads (which is
435 ## the correct thing to do). We catch that here rather than wait for
436 ## file verification to catch it.
437 ## @param file to check
438 ## @Stdout error message if file fails the check
439 ## @return 1 if file fails the check
440 ## @return 0 if file passes the check
441 #---------------------------------------------------------------------
442 function source_sanity() {
443 [[ "$1" ]] || return 0
444 [[ "$SUMMON_SANITY" == off ]] && return 0
445 local core_msg
446 core_msg="${PROBLEM_COLOR}Sanity check of${DEFAULT_COLOR}"
447 core_msg="$core_msg ${FILE_COLOR}$1${DEFAULT_COLOR}"
448 core_msg="$core_msg ${PROBLEM_COLOR}failed:${DEFAULT_COLOR}\n"
449
450 if ! test -s "$1"; then
451 message "$core_msg Empty"
452 return 1
453 fi
454
455 if [[ "$1" != "${1%.tar.???}" ]] ||
456 [[ "$1" != "${1%.tar.??}" ]] ||
457 [[ "$1" != "${1%.tar.?}" ]] ||
458 [[ "$1" != "${1%.tar}" ]] ||
459 [[ "$1" != "${1%.zip}" ]] ||
460 [[ "$1" != "${1%.tgz}" ]] ||
461 [[ "$1" != "${1%.sig}" ]] ||
462 [[ "$1" != "${1%.sign}" ]] ||
463 [[ "$1" != "${1%.asc}" ]]; then
464 local _type=$(file -bi "$1")
465 if echo $_type | grep -iq "text/"; then
466 message "$core_msg $_type"
467 return 1
468 fi
469 fi
470
471 }
472 #---------------------------------------------------------------------
473 ## This software is free software; you can redistribute it and/or modify
474 ## it under the terms of the GNU General Public License as published by
475 ## the Free Software Foundation; either version 2 of the License, or
476 ## (at your option) any later version.
477 ##
478 ## This software is distributed in the hope that it will be useful,
479 ## but WITHOUT ANY WARRANTY; without even the implied warranty of
480 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
481 ## GNU General Public License for more details.
482 ##
483 ## You should have received a copy of the GNU General Public License
484 ## along with this software; if not, write to the Free Software
485 ## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
486 #---------------------------------------------------------------------