blob: be52ee3d862cea4e898b1de8df258a69652d4381 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
|
#!/bin/sh
#
# Parse stagit-style repos
#
if [ $# -lt 2 ]; then
printf "Usage: %s <urlbase> <destdir>\n" $0
exit 1
fi
FIN="/dev/stdin"
URLBASE="$1"
DEST="$2"
SUBPATH="/file"
CURL="torify curl -Ls "
## func
cleanup () {
rm -f $TMPFILE
exit
}
## func
get_repos() {
repos=$(cat $TMPFILE | grep "/html/body/div/table/tbody/tr/td/a" \
| awk '{print $(NF-1), $NF}' \
| sed -E 's/href=//g;s/ /\|/'\
)
}
## func
get_descr() {
reponame=$1
echo "reponame: $reponame"
## Get author and description
cat "$TMPFILE" | grep -Ei -A 2 "/html/body/div/table/tbody/tr/td/a[[:blank:]]+href=${reponame}/log.html" |\
tail -2 | cut -f 2-
}
## func
get_readmes() {
LINK=$1
DESTDIR=$2
for f in $READMES; do
printf " trying file %s..." $LINK/$f
$CURL "$LINK/$f" > $DESTDIR/$f.tmp
failure=$(xml2tsv < $DESTDIR/$f.tmp 2>/dev/null | \
grep -Eaic "^/html/head/title[[:blank:]]+404 Not Found")
echo $failure
if [ "$failure" = 1 ]; then
printf "[FAILED]\n"
else
xml2tsv < $DESTDIR/$f.tmp 2>/dev/null | \
grep -Eai "/html/body/div/pre/a[[:blank:]]+href=#.*[[:blank:]]+class=line" | \
cut -f 6- | \
sed -E 's/\\n//g;s/\\t/\t/g;s/\\\\/\\/g' > $DESTDIR/$f
printf "[OK]\n"
fi
rm -f $DESTDIR/$f.tmp
sleep 1
done
}
# main loop
PROTO=${URLBASE%%:\/\/*}
DIRBASE="$PROTO/${URLBASE##[a-z]*:\/\/}"
echo "proto: $PROTO"
echo "dirbase: $DIRBASE"
READMES="README.html README.txt.html README.md.html readme.html readme.txt.html readme.md.html"
TMPFILE="./tmp_$$"
trap cleanup EXIT KILL TERM INT
$CURL "${URLBASE}" | xml2tsv > $TMPFILE
get_repos
for r in $repos; do
name=$(echo "$r" | cut -d "|" -f 2 )
link=$(echo "$r" | cut -d "|" -f 1 )
link="${link%%log.html}"
baselink=$(printf "%s/%s" $URLBASE $link)
printf "link: %s\nbaselink: %s\n" $link $baselink 1>&2
REPODIR="$DEST/$DIRBASE/$link/"
mkdir -p $REPODIR
get_descr $name > ${REPODIR}/DESCR
## Get READMEs
get_readmes "$baselink/$SUBPATH" "$REPODIR"
done
cleanup
|
