xcache.singularity.def

#
# Author: Wei Yang (SLAC National Accelerator Laboratory / Stanford University, 2017)
#

Bootstrap: yum
OSVersion: 7
MirrorURL: http://mirror.centos.org/centos-%{OSVERSION}/%{OSVERSION}/os/$basearch/
Include: yum

%labels
Maintainer Wei.Yang
Version Xcache-4.7.1,rucioN2N-1.0
Purpose Xcache-for-RUCIO

%setup
  mkdir -p $SINGULARITY_ROOTFS/etc/grid-security/certificates 
  mkdir -p $SINGULARITY_ROOTFS/etc/grid-security/vomsdir

  # $GRIDSECURITY is a directory on container local build host
  # GRIDSECURITY=/etc/grid-security

  GRIDSECURITY="/cvmfs/oasis.opensciencegrid.org/mis/osg-wn-client/current/el7-x86_64/etc/grid-security"
  if [ -d $GRIDSECURITY ]; then
    cd $GRIDSECURITY
    tar chf - certificates vomsdir | (cd $SINGULARITY_ROOTFS/etc/grid-security; tar xf -)
  fi

%post
  yum install -y curl gperftools hostname

  curl -s -o /etc/yum.repos.d/xrootd-stable-slc7.repo http://www.xrootd.org/binaries/xrootd-stable-slc7.repo
  curl -s -o /etc/pki/rpm-gpg/RPM-GPG-KEY-wlcg http://linuxsoft.cern.ch/wlcg/RPM-GPG-KEY-wlcg
  curl -s -o /etc/yum.repos.d/wlcg-centos7.repo http://linuxsoft.cern.ch/wlcg/wlcg-centos7.repo

  yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
  yum install -y xrootd-server xrootd-client xrootd xrootd-debuginfo
  yum install -y xrootd-rucioN2N-for-Xcache
  yum install -y vomsxrd
  yum install -y gdb
  mkdir -p /data 

  echo "g /atlas / rl" > /etc/xrootd/auth_db
  mkdir -p /etc/grid-security/xrd
  touch /etc/grid-security/xrd/xrdcert.pem
  touch /etc/grid-security/xrd/xrdkey.pem

  touch /etc/xrootd/xcache.cfg /var/run/x509up
  cat > /etc/xrootd/xcache.cfg.template <<EOF
# "redirector" should be full qualified DNS name (e.g. hostname -f)
set redirector = XCACHE_RDR

all.manager \$(redirector):1213
all.adminpath /data/xrd/var/spool/xrootd
all.pidpath   /data/xrd/var/run/xrootd

oss.localroot  /data/xrd/namespace

all.export /atlas/rucio stage r/o
all.export /root:/      stage r/o
all.export /xroot:/     stage r/o

if \$(redirector)
    all.role manager
    cms.delay startup 10
else if exec cmsd
    all.role server
    oss.statlib /usr/lib64/XrdName2NameDCP4RUCIO.so
else
    oss.space meta /data/xrd/xrdcinfos
    oss.space data /data/xrd/datafiles

    all.role server
    oss.path /atlas/rucio r/w
    oss.path /root:/      r/w
    oss.path /xroot:/     r/w
    ofs.osslib   /usr/lib64/libXrdPss.so
    pss.cachelib /usr/lib64/libXrdFileCache.so
    pss.config streams 128
    pss.origin localfile:1094
    pss.namelib -lfncache -lfn2pfn /usr/lib64/XrdName2NameDCP4RUCIO.so

    pfc.ram XCACHE_RAMSIZE
    pfc.diskusage XCACHE_SPACE_LO_MARK XCACHE_SPACE_HI_MARK
    pfc.spaces data meta
    pfc.blocksize 1M
    pfc.prefetch 0
    pfc.trace info

    # Uncomment the following and bind mount to /etc/xrootd/xcache.cfg to request GSI security from client
    #xrootd.seclib /usr/lib64/libXrdSec.so
    #sec.protparm gsi -vomsfun:/usr/lib64/libXrdSecgsiVOMS.so -vomsfunparms:certfmt=raw|vos=atlas|grps=/atlas
    #sec.protocol /usr/lib64 gsi -ca:1 -crl:3
    #acc.authdb /etc/xrootd/auth_db
    #acc.authrefresh 60
    #ofs.authorize
fi
EOF

%runscript
# X509_USER_PROXY, X509_CERT_DIR, X509_VOMS_DIR do not have to be defined/provided

# if x509 user proxy is provided in a non-standard location (/tmp/x509up_u$(id -u)), 
# then the proxy should be bind mounted: -B ${X509_USER_PROXY}:/var/run/x509up

unset X509_USER_PROXY
[ -s /var/run/x509up ] && export X509_USER_PROXY=/var/run/x509up

# if X509_CERT_DIR is not defined, or is inaccessible in the container, then we use
# the default location. Same for X509_VOMS_DIR.
# One can also bind mount:
#     -B ${X509_CERT_DIR}:/etc/grid-security/certificates
#     -B ${X509_VOMS_DIR}:/etc/grid-security/vomsdir

[ ! -z "$X509_CERT_DIR" ] && [ ! -d "$X509_CERT_DIR" ] && export X509_CERT_DIR=/etc/grid-security/certificates
[ ! -z "$X509_VOMS_DIR" ] && [ ! -d "$X509_VOMS_DIR" ] && export X509_VOMS_DIR=/etc/grid-security/vomsdir 
 
mkdir -p /data/xrd/namespace /data/xrd/xrdcinfos /data/xrd/datafiles
mkdir -p /data/xrd/var/log /data/xrd/var/spool /data/xrd/var/run

runcmsd=0
if [ -z "$XCACHE_RDR" ]; then
  XCACHE_RDR="www.google.com" # XCACHE_RDR must be defined
else
  runcmsd=1
fi
if [ -z "$XCACHE_PFCRAM" ]; then
  XCACHE_PFCRAM=$(free | tail -2 | head -1 | awk '{printf("%d", $NF/1024/1024/2)}')
  [ $XCACHE_PFCRAM -lt 1 ] && XCACHE_PFCRAM=1
  XCACHE_PFCRAM=${XCACHE_PFCRAM}g
fi
[ -z "$XCACHE_SPACE_LO_MARK" ] && XCACHE_SPACE_LO_MARK="0.75"
[ -z "$XCACHE_SPACE_HI_MARK" ] && XCACHE_SPACE_HI_MARK="0.85"

if [ -s /etc/xrootd/xcache.cfg ]; then
  xcache_cfg=/etc/xrootd/xcache.cfg
else
  xcache_cfg=/tmp/xcache.cfg
  cat > $xcache_cfg <<EOF
# Xcache config file - this file is automatically generated
# Modification of this file will be lost.
#
# To make a persistant change, modify and save to another file and then 
# bind mount the new config file to /etc/xrootd/xcache.cfg

# Info about the system:

EOF

  free | sed -e 's/^/\#\ /g' >> $xcache_cfg
  echo "" >> $xcache_cfg
  df -k /data | sed -e 's/^/\#\ /g' >> $xcache_cfg
  echo "" >> $xcache_cfg
  cat /etc/xrootd/xcache.cfg.template | sed -e "s/XCACHE_SPACE_LO_MARK/$XCACHE_SPACE_LO_MARK/g" | \
                                        sed -e "s/XCACHE_SPACE_HI_MARK/$XCACHE_SPACE_HI_MARK/g" | \
                                        sed -e "s/XCACHE_RAMSIZE/$XCACHE_PFCRAM/g" | \
                                        sed -e "s/XCACHE_RDR/$XCACHE_RDR/g" >> $xcache_cfg
fi

#echo $X509_USER_PROXY $X509_CERT_DIR $X509_VOMS_DIR

export LD_PRELOAD=/usr/lib64/libtcmalloc.so
export TCMALLOC_RELEASE_RATE=10
export XRD_LOCALMETALINKFILE=1
/usr/bin/xrootd -b -c $xcache_cfg -l /data/xrd/var/log/xrootd.log -k hup
[ "$runcmsd" -eq 1 ] && /usr/bin/cmsd -b -c $xcache_cfg -l /data/xrd/var/log/cmsd.log -k 7