Difference between revisions of "User:AskApache/pacman-rankmirrors"

From ArchWiki
Jump to: navigation, search
(rm cat from user page)
(new version)
Line 1: Line 1:
 
This shell script shows a better way to rank your pacman mirrors, especially for powerpill setup to use aria2c.
 
This shell script shows a better way to rank your pacman mirrors, especially for powerpill setup to use aria2c.
 
+
{{ic|<nowiki>~/scripts/reflector.sh  | sudo tee /etc/pacman.d/mirrorlist</nowiki>}}
  
 
<pre><nowiki>
 
<pre><nowiki>
 
#!/bin/bash
 
#!/bin/bash
 +
# Updated: Tues May 07 21:04:12 2013 by webmaster@askapache
 +
# @ http://www.askapache.com/linux/reflector-ranking-mirrors.html
 +
# Copyright (C) 2013 Free Software Foundation, Inc.
 +
#
 +
#  This program is free software: you can redistribute it and/or modify
 +
#  it under the terms of the GNU General Public License as published by
 +
#  the Free Software Foundation, either version 3 of the License, or
 +
#  (at your option) any later version.
 +
#
 +
#  This program is distributed in the hope that it will be useful,
 +
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
 +
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 +
#  GNU General Public License for more details.
 +
#
 +
#  You should have received a copy of the GNU General Public License
 +
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  
  
# i686
 
UNAME=`uname -m`
 
 
ARCHFTP=ftp://ftp.archlinux.org/core/os/$UNAME/
 
 
# used to prevent multiple downloads of mirrorlist
 
MIRROR=0
 
 
# get gcc url
 
GCCFILE=core/os/${UNAME}/$(curl -H --url $ARCHFTP 2>/dev/null|awk '{print $5,$9}'|sort -nr -k5|grep -o gcc-.*xz -m 1)
 
 
# passed to async curl
 
CURLOPTS='curl -o /dev/null --connect-timeout 10 --retry 2 --no-keepalive -w "%{speed_download} %{url_effective}\n" -s'
 
 
 
get_mirrorlist_url()
 
{ curl -Hl --url $ARCHFTP 2>/dev/null|sed -n 's/^.* \(pacman-mirrorlist.*gz\) .*$/\1/p' 2>/dev/null; }
 
  
get_mirrorlist()
+
# if mirrors exists, cat it, otherwise create it
{ [[ $MIRROR -ne 1 ]] && { curl -o - $ARCHFTP`get_mirrorlist_url` | tar -xzf - -O etc/pacman.d/mirrorlist > $0.mirrorlist && export MIRROR=1; }; cat $0.mirrorlist; }
+
function get_mirrors () #{{{1
 +
{
 +
  if [[ -s $MIRRORS ]]; then
 +
          cat $MIRRORS;
 +
  else
 +
          curl -LksS -o - 'https://www.archlinux.org/mirrors/status/json/' | sed 's,{,\n{,g' | sed -n '/rsync/d; /pct": 1.0/p' | sed 's,^.*"url": "\([^"]\+\)".*,\1,g' > $MIRRORS
 +
          cat $MIRRORS;
 +
  fi
 +
}
  
get_servers ()  
+
function get_core_urls () #{{{1
{ get_mirrorlist|sed -e '/Ser/s,^.*erver.*=[^:]*://\([^/]*\)/.*$,\1,g' | sort -u; }
+
{
 +
  get_mirrors | sed "s,$,core/os/${ARCH}/core.db.tar.gz,g"
 +
}
  
resolve_servers()
+
function get_gcc_urls () #{{{1
{ get_servers|xargs -I'{}' -P10 sh -c "ping -c1 -q '{}' 2>&1 | sed -n '/bytes/p'"; }
+
{
 +
  get_mirrors | sed "s,$,core/os/${ARCH}/${GCC_URL},g"
 +
}
  
get_server_urls()
 
{ get_mirrorlist | sed -e '/Ser/s,^.*erver.*= *\(.*://[^$]*\)/$repo.*$,\1/,g;/:\/\//!d'; }
 
  
pmsg ()
+
# rm tmp file on exit
{ echo -e "\n\n`tput rev`$@`tput sgr0`"; }
+
trap "exitcode=\$?; (rm -f \$MIRRORS 2>/dev/null;) && exit \$exitcode" 0;
 +
trap "exit 1" 1 2 13 15;
  
  
# blanking files
+
# file containing mirror urls
echo > $0.1.log
+
MIRRORS=`(mktemp -t reflector-mirrorsXXXX) 2>/dev/null` && test -w "$MIRRORS" || MIRRORS=~/reflector.mirrorsXXX
echo > $0.2.log
+
echo > $0.mirrorlist
+
echo > optimized-mirrorlist
+
  
pmsg "Resolving Servers to minimize DNS lookups interfering with speed tests"
+
# arch
resolve_servers
+
ARCH=`(uname -m) 2>/dev/null` || ARCH=x86_64
  
pmsg "Timing each servers speed for a large file '${GCCFILE}'"
+
# the gcc file
get_server_urls | xargs -I'{}' -P5 sh -c "$CURLOPTS --url {}${GCCFILE}" | tee -a $0.2.log
+
GCC_URL=$( curl -LksSH --url ftp://ftp.archlinux.org/core/os/${ARCH}/ 2>/dev/null | sed -n 's/^.*\ \(gcc-[0-9]\+.*.tar.xz.sig\)\ -.*$/\1/gp' );
sort -nr $0.2.log|sed 45q | sed 's/^.* \(.*\)$/Server = \1/g' | tee -a optimized-mirrorlist
+
  
pmsg "Timing each servers speed for a small file 'core.db.gz'"
 
get_server_urls | xargs -I'{}' -P5 $CURLOPTS -m 20 --url "{}core/os/${UNAME}/core.db.gz" | tee -a $0.1.log
 
sort -nr $0.1.log|sed 3q |sed 's/^.* \(.*\)$/Server = \1/g'| tee -a optimized-mirrorlist
 
  
pmsg "Optimized Mirrorlist at 'optimized-mirrorlist'"
+
{
sort -u optimized-mirrorlist
+
  # faster as primarily used to pre-resolve dns for 2nd core test
 +
  get_gcc_urls | xargs -I'{}' -P40 curl -Lks -o /dev/null -m 3 --connect-timeout 4 --retry 0 --no-keepalive -w '%{time_total}@%{speed_download}@%{url_effective}\n' --url '{}' | sort -t@ -k2 -nr | head -n 50 | cut -d'@' -f3 | sed 's,core/os/'"${ARCH}/${GCC_URL}"',$repo/os/$arch,g'
 +
  get_core_urls | xargs -I'{}' -P10 curl -Lks -o /dev/null -m 5 --connect-timeout 4 --retry 0 --no-keepalive -w '%{time_total}@%{speed_download}@%{url_effective}\n' --url '{}' | sort -t@ -k2 -nr | head -n 50 | cut -d'@' -f3 | sed 's,core/os/'"${ARCH}"'/core.db.tar.gz,$repo/os/$arch,g'
 +
} | sed 's,^,Server = ,g' | awk '{ if (!h[$0]) { print $0; h[$0]=1 } }'
  
rm -v $0.1.log $0.2.log $0.mirrorlist
 
  
exit $?
+
exit $?;
 
</nowiki>
 
</nowiki>
 
</pre>
 
</pre>

Revision as of 06:06, 7 May 2013

This shell script shows a better way to rank your pacman mirrors, especially for powerpill setup to use aria2c. ~/scripts/reflector.sh | sudo tee /etc/pacman.d/mirrorlist

#!/bin/bash
# Updated: Tues May 07 21:04:12 2013 by webmaster@askapache
# @ http://www.askapache.com/linux/reflector-ranking-mirrors.html
# Copyright (C) 2013 Free Software Foundation, Inc.
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.



# if mirrors exists, cat it, otherwise create it
function get_mirrors () #{{{1
{
   if [[ -s $MIRRORS ]]; then
          cat $MIRRORS;
   else
          curl -LksS -o - 'https://www.archlinux.org/mirrors/status/json/' | sed 's,{,\n{,g' | sed -n '/rsync/d; /pct": 1.0/p' | sed 's,^.*"url": "\([^"]\+\)".*,\1,g' > $MIRRORS
          cat $MIRRORS;
   fi
}

function get_core_urls () #{{{1
{
   get_mirrors | sed "s,$,core/os/${ARCH}/core.db.tar.gz,g"
}

function get_gcc_urls () #{{{1
{
   get_mirrors | sed "s,$,core/os/${ARCH}/${GCC_URL},g"
}


# rm tmp file on exit
trap "exitcode=\$?; (rm -f \$MIRRORS 2>/dev/null;) && exit \$exitcode" 0;
trap "exit 1" 1 2 13 15;


# file containing mirror urls
MIRRORS=`(mktemp -t reflector-mirrorsXXXX) 2>/dev/null` && test -w "$MIRRORS" || MIRRORS=~/reflector.mirrorsXXX

# arch
ARCH=`(uname -m) 2>/dev/null` || ARCH=x86_64

# the gcc file
GCC_URL=$( curl -LksSH --url ftp://ftp.archlinux.org/core/os/${ARCH}/ 2>/dev/null | sed -n 's/^.*\ \(gcc-[0-9]\+.*.tar.xz.sig\)\ -.*$/\1/gp' );


{
   # faster as primarily used to pre-resolve dns for 2nd core test
   get_gcc_urls | xargs -I'{}' -P40 curl -Lks -o /dev/null -m 3 --connect-timeout 4 --retry 0 --no-keepalive -w '%{time_total}@%{speed_download}@%{url_effective}\n' --url '{}' | sort -t@ -k2 -nr | head -n 50 | cut -d'@' -f3 | sed 's,core/os/'"${ARCH}/${GCC_URL}"',$repo/os/$arch,g'
   get_core_urls | xargs -I'{}' -P10 curl -Lks -o /dev/null -m 5 --connect-timeout 4 --retry 0 --no-keepalive -w '%{time_total}@%{speed_download}@%{url_effective}\n' --url '{}' | sort -t@ -k2 -nr | head -n 50 | cut -d'@' -f3 | sed 's,core/os/'"${ARCH}"'/core.db.tar.gz,$repo/os/$arch,g'
} | sed 's,^,Server = ,g' | awk '{ if (!h[$0]) { print $0; h[$0]=1 } }'


exit $?;