I have a 500 line program I wrote that reads openbsd.org.ftp.html and scraps off the html and ftp mirrors, records them all without redundancies as http mirrors in memory and downloads the appropriate version and machine architecture's SHA256 in the package folder. It tests all the mirrors for time, one at a time and uses kqueue to kill any laggy ftp calls. It uses ftp() calls for all its networking, so it shouldn't be too much of a security issue I'd guess. It writes the top 8 mirrors into /etc/pkg.conf it erases all the installpath entries while leaving everything else in the file. It can run as an unprivileged user, but of course it won't rewrite /etc/pkg.conf -Luke N Small
/* * Copyright (c) 2016 Luke N. Small * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
/* Special thanks to Dan Mclaughlin for the ftp to sed idea * * ftp -o - http://www.openbsd.org/ftp.html | \ * sed -n \ * -e 's: <strong>\([^<]*\)<.*:\1 :p' \ * -e 's:^\( [hfr].*\):\1:p' */ #define EVENT_NOPOLL #define EVENT_NOSELECT #include <stdio.h> #include <unistd.h> #include <stdlib.h> #include <err.h> #include <sys/types.h> #include <sys/event.h> #include <signal.h> #include <string.h> struct mirror_st { char * countryTitle; char * mirror; char * installPath; double diff; struct mirror_st * next; }; int ftp_cmp (const void * a, const void * b) { struct mirror_st **one = (struct mirror_st **)a; struct mirror_st **two = (struct mirror_st **)b; if( (*one)->diff < (*two)->diff ) return -1; if( (*one)->diff > (*two)->diff ) return 1; return 0; } int country_cmp (const void * a, const void * b) { struct mirror_st **one = (struct mirror_st **)a; struct mirror_st **two = (struct mirror_st **)b; // list the USA mirrors first, it will subsort correctly int8_t temp = !strncmp("USA", (*one)->countryTitle, 3); if(temp != !strncmp("USA", (*two)->countryTitle, 3)) { if(temp) return -1; return 1; } return strcmp( (*one)->countryTitle, (*two)->countryTitle ) ; } double getTimeDiff(struct timeval a, struct timeval b) { long sec; long usec; sec = b.tv_sec - a.tv_sec; usec = b.tv_usec - a.tv_usec; if (usec < 0) { --sec; usec += 1000000; } return sec + ((double)usec / 1000000.0); } int main() { pid_t ftpPid, sedPid; int ftpToSed[2]; int sedToParent[2]; int unameToParent[2]; char unameR[5], unameM[20]; int i = 0, c; register int position, num; FILE *input; pipe(unameToParent); // "uname -rm" returns version and architecture like: "5.8 amd64" to standard out if(fork() == (pid_t) 0) { /* uname child */ close(unameToParent[0]); dup2(unameToParent[1], STDOUT_FILENO); /*attaching to pipe(s)*/ execl("/usr/bin/uname","/usr/bin/uname", "-rm", NULL); err(1, "execl() failed\n"); } close(unameToParent[1]); input = fdopen (unameToParent[0], "r"); num = 0; position = -1; while ((c = getc(input)) != EOF) { if(num == 0) { if(c != ' ') unameR[++position] = c; else { unameR[position + 1] = '\0'; num = 1; position = -1; } } else { if(c != '\n') unameM[++position] = c; else unameM[position + 1] = '\0'; } } fclose (input); close(unameToParent[0]); pipe(ftpToSed); /*make pipes*/ struct kevent ke[2]; int kq = kqueue(); if (kq == -1) err(1, "kq!"); int kqProc = kqueue(); if (kqProc == -1) err(1, "kqProc!"); ftpPid = fork(); if(ftpPid == (pid_t) 0) { /*ftp child*/ close(ftpToSed[0]); dup2(ftpToSed[1], STDOUT_FILENO); /*attaching to pipe(s)*/ execl("/usr/bin/ftp","ftp", "-Vo", "-", "http://www.openbsd.org/ftp.html";, NULL); err(1, "execl() failed\n"); } EV_SET(ke, ftpPid, EVFILT_PROC, EV_ADD | EV_ONESHOT, NOTE_EXIT, 0, &ftpPid); if (kevent(kqProc, ke, 1, NULL, 0, NULL) == -1) err(1, "kevent register fail."); close(ftpToSed[1]); pipe(sedToParent); sedPid = fork(); if(sedPid == (pid_t) 0) { /* sed child */ close(sedToParent[0]); dup2(ftpToSed[0], STDIN_FILENO); /*attaching to pipe(s)*/ dup2(sedToParent[1], STDOUT_FILENO); execl("/usr/bin/sed","sed","-n","-e", "s:\t<strong>\\([^<]*\\)<.*:\\1 :p", "-e", "s:^\\(\t[hfr].*\\):\\1:p", NULL); kill(ftpPid, SIGKILL); err(1, "execl() failed\n"); } EV_SET(ke, sedPid, EVFILT_PROC, EV_ADD | EV_ONESHOT, NOTE_EXIT, 0, &sedPid); if (kevent(kqProc, ke, 1, NULL, 0, NULL) == -1) err(1, "kevent register fail."); close(ftpToSed[0]); /*close pipe(s) child attached to*/ close(sedToParent[1]); EV_SET(ke, sedToParent[0], EVFILT_READ, EV_ADD | EV_ONESHOT, 0, 0, &sedToParent[0]); if (kevent(kq, ke, 1, NULL, 0, NULL) == -1) err(1, "kevent register fail."); struct timespec timeout; timeout.tv_sec = 7; timeout.tv_nsec = 0; i = kevent(kq, NULL, 0, ke, 1, &timeout); if (i == -1) err(1, "kevent"); if(i == 0) { close(sedToParent[0]); printf("timed out.\n"); kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); return -1; } input = fdopen (sedToParent[0], "r"); char line[200]; num = 0; position = 0; struct mirror_st *start, *end, *mTemp1, *mTemp2, *mTemp3; start = end = malloc(sizeof(struct mirror_st)); start->next = NULL; while((c = getc (input)) != EOF) { if(num == 0) { if( c != '\n' ) line[position++] = c; else { // there is a space before the newline that is eliminated line[position-1] = '\0'; end->countryTitle = malloc(position); strlcpy(end->countryTitle, line, position); position = 0; num = 1; } } else { if(position == 0) { if( (c != 'h') && (c != 'f') && (c != 'r') ) continue; else if(c == 'r') break; else if(c == 'f') { // changes ftp listings to http. ftp.html says they can be either line[position++] = 'h'; line[position++] = 't'; continue; } } if( c != '\n' ) line[position++] = c; else { // there is a "</a>" before the newline that is overwritten line[position -= 4] = '\0'; ++position; position += num = strlen(unameR) + 10 + strlen(unameM) + 7; end->mirror = malloc(position); c = strlcpy(end->mirror, line, position); c += strlcpy(end->mirror + c, unameR, position - c); c += strlcpy(end->mirror + c, "/packages/", position - c); c += strlcpy(end->mirror + c, unameM, position - c); strlcpy(end->mirror + c, "/SHA256", position - c); position += 15 - num; end->installPath = malloc(position); c = strlcpy(end->installPath, line, position); strlcpy(end->installPath + c, "%c/packages/%a/", position - c); end = end->next = malloc(sizeof(struct mirror_st)); end->next = NULL; position = 0; num = 0; } } } mTemp1 = start; while(mTemp1->next != end) mTemp1 = mTemp1->next; free(end->countryTitle); free(end); end = mTemp1; end->next = NULL; mTemp1 = start; while(mTemp1->next != NULL) { mTemp2 = mTemp1; do { if(!strcmp(mTemp1->mirror, mTemp2->next->mirror)) { mTemp3 = mTemp2->next; mTemp2->next = mTemp3->next; free(mTemp3->countryTitle); free(mTemp3->installPath); free(mTemp3->mirror); free(mTemp3); } mTemp2 = mTemp2->next; if(mTemp2 == NULL) break; } while( mTemp2->next != NULL ); mTemp1 = mTemp1->next; } int mirrorNum = 0; mTemp1 = start; while(mTemp1 != NULL) { ++mirrorNum; mTemp1 = mTemp1->next; } struct mirror_st ** Array; if( (Array = calloc(mirrorNum, sizeof(struct mirror_st *))) == NULL) err(1, "malloc failed."); mTemp1 = start; for(c = 0; c < mirrorNum; ++c) { Array[c] = mTemp1; mTemp1 = mTemp1->next; } qsort(Array, mirrorNum, sizeof(struct mirror_st *), country_cmp); fclose (input); close(sedToParent[0]); i = kevent(kqProc, NULL, 0, ke, 2, &timeout); if (i == -1) err(1, "kevent"); if(i == 0) { printf("timed out.\n"); kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); return -1; } else if(i == 1) { printf("timed out.\n"); if( ( (int *)ke->udata ) == &sedPid) kill(ftpPid, SIGKILL); else kill(sedPid, SIGKILL); return -1; } close(kq); close(kqProc); kqProc = kqueue(); if (kqProc == -1) err(1, "kqProc!"); int pid; for(c = 0; c < mirrorNum; ++c) { mTemp1 = Array[c]; position = 0; num = -1; printf("\n%d : %s : %s\n", (mirrorNum - c) - 1, mTemp1->countryTitle, mTemp1->mirror); pid = fork(); if(pid == (pid_t) 0) { /* ping child */ execl("/usr/bin/ftp", "ftp", "-Vmo", "/dev/null", mTemp1->mirror, NULL); err(1, "execl() failed\n"); } EV_SET(ke, pid, EVFILT_PROC, EV_ADD | EV_ONESHOT, NOTE_EXIT, 0, NULL); if (kevent(kqProc, ke, 1, NULL, 0, NULL) == -1) err(1, "kevent register fail."); struct timeval tv_start, tv_end; gettimeofday(&tv_start, NULL); skip: i = kevent(kqProc, NULL, 0, ke, 1, &timeout); if (i == -1) err(1, "kevent"); if(i == 0) { printf("\n"); kill(pid, SIGKILL); mTemp1->diff = timeout.tv_sec + (double)timeout.tv_nsec / 1000000000.0; goto skip; } if(ke->data != 0 || mTemp1->diff == (timeout.tv_sec + (double)timeout.tv_nsec / 1000000000.0) ) mTemp1->diff = timeout.tv_sec + (double)timeout.tv_nsec / 1000000000.0; else { gettimeofday(&tv_end, NULL); mTemp1->diff = getTimeDiff(tv_start, tv_end); } } qsort(Array, mirrorNum, sizeof(struct mirror_st *), ftp_cmp); for(c = mirrorNum - 1; c >= 0; --c) { Array[c]->mirror[strlen(Array[c]->mirror) - 6] = '\0'; printf("%d : %s:\n\t%s: %f\n\n", c, Array[c]->countryTitle, Array[c]->installPath, Array[c]->diff); } char *buf; int lines; if( (input = fopen("/etc/pkg.conf", "r")) == NULL ) { buf = (char*)malloc(800); c = strlcpy(buf, "installpath = ", 800); c += strlcpy(buf + c, Array[0]->installPath, 800 - c); for(position = 1; position < 8; ++position) { c += strlcpy(buf + c, "\ninstallpath += ", 800 - c); c += strlcpy(buf + c, Array[position]->installPath, 800 - c); } } else { fseek(input, 0, SEEK_END); num = ftell(input); fseek(input, 0, SEEK_SET); lines = 0; buf = (char*)malloc(num + 800); fread(buf, 1, num, input); fclose(input); for(c = 0; c < num; ++c) { if(buf[c] == '\n') ++lines; } position = 0; int copy = 0; for(c = 0; c < num; ++c) { if(buf[c] == '\n') { --lines; position = 0; } else if(position++ == 0) { if( !strncmp(buf + c, "installpath", 11) ) { if(lines) { while(buf[++c] != '\n'); --lines; position = 0; continue; } else break; } } buf[copy++] = buf[c]; } buf[copy] = '\0'; num += 800; c = copy + strlcpy(buf + copy, "installpath = ", num - copy); c += strlcpy(buf + c, Array[0]->installPath, num - c); for(position = 1; position < 8; ++position) { c += strlcpy(buf + c, "\ninstallpath += ", num - c); c += strlcpy(buf + c, Array[position]->installPath, num - c); } } input = fopen("/etc/pkg.conf", "w"); if(input != NULL) { fwrite(buf, 1, c, input); fclose(input); printf("\n\nedit out all PKG_PATH environment exports and run \"unset PKG_PATH\".\n\n"); printf("/etc/pkg.conf:\n%s\n", buf); } else { printf("\n\nThis could have been the contents of /etc/pkg.conf (run as superuser):\n"); printf("%s\n", buf); } return 0; }
