The following commit has been merged in the master branch: commit 17c762e952d7fa4c4e1499afdb9250e5530ccf47 Author: Andreas Tille <ti...@debian.org> Date: Sun Feb 11 17:53:51 2018 +0100
Start working on fetching machine readable data from salsa diff --git a/misc/machine_readable/fetch-machine-readable_salsa b/misc/machine_readable/fetch-machine-readable_salsa new file mode 100755 index 0000000..4f244d3 --- /dev/null +++ b/misc/machine_readable/fetch-machine-readable_salsa @@ -0,0 +1,92 @@ +#!/bin/sh +# This script should be called on Alioth to gather all machine readable files +# of VCSes mentioned in SVNDIRS and GITDIRS +# +# TODO: store file <pkg>.vcs with the Vcs URL where the files were found + +. ./salsarc + +TDNAME=machine-readable +MACHINEREADABLEARCHIVE=/srv/blends.debian.org/www/_${TDNAME}/${TDNAME}.tar.bz2 +READMEDEBIANARCHIVE=/srv/blends.debian.org/www/_${TDNAME}/README.Debian.tar.bz2 +TARGETDIR=${HOME}/$TDNAME +mkdir -p $TARGETDIR + +ERRLOG=${HOME}/${TDNAME}.log +rm -f $ERRLOG + +GITTEAMS="debichem-team + med-team + r-pkg-team + science-team" + +ANONSCM_DICT="debichem-team:debichem/packages\n + med-team:debian-med\n + r-pkg-team:debian-science/packages\n + science-team:debian-science/packages" + +get_one_page () { + curl --silent -D $HEADERS_FILE "$SALSA_URL/groups/$SALSA_GROUP_ID/projects?private_token=$SALSA_TOKEN&simple=true&per_page=$PER_PAGE&page=$PAGENO" | jq --raw-output ".[] | .path" +} + +list_salsa_projects () { + SALSA_GROUP="$1" + PER_PAGE=100 + HEADERS_FILE=$(mktemp) + trap "rm -f $HEADERS_FILE" EXIT + + SALSA_GROUP_ID=$(curl --silent -f -XGET --header "PRIVATE-TOKEN: $SALSA_TOKEN" "$SALSA_URL/groups?all_available=false" | jq ".[] | select(.path == \"$SALSA_GROUP\") | .id") + + PAGENO=1 + + get_one_page + + TOTAL_PAGES=$(grep ^X-Total-Pages: $HEADERS_FILE) + TOTAL_PAGES=${TOTAL_PAGES#*: } + + while [ $PAGENO -lt $TOTAL_PAGES ] + do + PAGENO=$((PAGENO+1)) + get_one_page + done +} + +mkdir -p $TARGETDIR +rm -rf $TARGETDIR/* + +for team in $GITTEAMS ; do + echo $team >> $ERRLOG + for gitrepo in `list_salsa_projects $team` ; do + firstletter=`echo $gitrepo | sed "s/^\(.\).*/\1/"` + mkdir -p $TARGETDIR/$firstletter + echo "Vcs-Git: g...@salsa.debian.org:$team/$gitrepo.git" > $TARGETDIR/$firstletter/${gitrepo}.vcs + # Hint: Debian Med and Debichem R packages do not match this but who cares ;-) + # Also some packages that never were on Alioth Git do not have a valid Vcs-Browser via that method + anonscmdir=`echo $ANONSCM_DICT | grep $team | head -n 1 | sed "s/^ *${team}://"` + echo "Vcs-Browser: https://anonscm.debian.org/cgit/$anonscmdir/$gitrepo.git" >> $TARGETDIR/$firstletter/${gitrepo}.vcs + wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/changelog -O $TARGETDIR/$firstletter/${gitrepo}.changelog + wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/control -O $TARGETDIR/$firstletter/${gitrepo}.control + wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/copyright -O $TARGETDIR/$firstletter/${gitrepo}.copyright + wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/upstream/metadata -O $TARGETDIR/$firstletter/${gitrepo}.upstream + wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/upstream/edam -O $TARGETDIR/$firstletter/${gitrepo}.edam + done +done + +FOUNDDIRS=`ls $TDNAME | wc -l` +if [ $FOUNDDIRS -lt 10 ] ; then + echo "Found way to less packages in $TDNAME" >> $ERRLOG + exit +fi +rm -f $MACHINEREADABLEARCHIVE +cd $TARGETDIR +cd .. +find $TDNAME -maxdepth 1 -mindepth 1 -name ".*" -delete +# Remove empty files +find $TDNAME -type f -empty -delete +tar --exclude=*README.Debian -cjf $MACHINEREADABLEARCHIVE $TDNAME +tar --exclude=*.control --exclude=*.changelog --exclude=*.upstream --exclude=*.edam --exclude=*.vcs --exclude=*.copyright -cjf $READMEDEBIANARCHIVE $TDNAME + +# Check for remaining tempfiles +if ls -l /tmp/tmp.* 2>/dev/null | grep `whoami` > /dev/null ; then + echo "There are tempfiles remaining!\n" | mailx -s "$0 has left some remaining files in /tmp" ti...@debian.org +fi -- Static and dynamic websites for Debian Pure Blends _______________________________________________ Blends-commit mailing list Blends-commit@lists.alioth.debian.org http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/blends-commit