Setting up IOFSL on Surveyor

Notice

replace "kota" with your user name!

Editing .bashrc

export SOFT_BASE=/home/kota/soft/
export PATH=$SOFT_BASE/bin:$PATH
export LD_LIBRARY_PATH=$SOFT_BASE/lib:$LD_LIBRARY_PATH

m4-1.4.13

• ​http://mirror.anl.gov/pub/gnu/m4/m4-1.4.13.tar.gz

  ./configure --prefix=$SOFT_BASE
  make
  make install
  

autoconf-2.64

• ​http://mirror.anl.gov/pub/gnu/autoconf/autoconf-2.64.tar.gz

  ./configure --prefix=$SOFT_BASE
  make
  make install
  

flex-2.5.35

• ​http://downloads.sourceforge.net/project/flex/flex/flex-2.5.35/flex-2.5.35.tar.gz?use_mirror=voxel

  ./configure --prefix=$SOFT_BASE
  make
  make install
  

bison-2.4

• ​ftp://mirror.anl.gov/pub/gnu/bison/bison-2.4.tar.gz

  ./configure --prefix=$SOFT_BASE
  make
  make install
  

automake-1.11

• ​ftp://mirror.anl.gov/pub/gnu/automake/automake-1.11.tar.gz

  ./configure --prefix=$SOFT_BASE
  make
  make install
  

ccache-2.4

• ​http://samba.org/ftp/ccache/ccache-2.4.tar.gz

  ./configure --prefix=$SOFT_BASE
  make
  make install
  

boost_1_40_0

• ​http://downloads.sourceforge.net/project/boost/boost/1.40.0/boost_1_40_0.tar.gz?use_mirror=superb-sea2

  ./bootstrap.sh --with-libraries=date_time,program_options,regex,thread,test --prefix=$SOFT_BASE
  ./bjam --prefix=$SOFT_BASE --libdir=$SOFT_BASE/lib --includedir=$SOFT_BASE/include install
  

CUnit-2.1-0

• ​http://downloads.sourceforge.net/project/cunit/CUnit/2.1-0/CUnit-2.1-0-src.zip?use_mirror=voxel

  ./configure --prefix=$SOFT_BASE
  make
  make install
  

openpa-1.0.2

• ​http://trac.mcs.anl.gov/projects/openpa/raw-attachment/wiki/Downloads/openpa-1.0.2.tar.gz

  ./configure --prefix=$SOFT_BASE
  make
  make install
  

BMI-ZOID (Checkout Only)

svn co https://svn.mcs.anl.gov/repos/ZeptoOS/trunk/BGP/packages/zoid/
export ZOID_DIR="point out the checked out zoid dir."

BMI + PVFS2 Library

We need libbmi (zoid + tcp) for communication, and libpvfs2 for dispatcher.

cvs -d :pserver:[email protected]:/anoncvs login
cvs -d :pserver:[email protected]:/anoncvs co pvfs2
cvs -d :pserver:[email protected]:/anoncvs logout
cd pvfs2
./configure --prefix=$SOFT_BASE --with-zoid=$ZOID_DIR/src --enable-bmi-only --without-bmi-tcp
make clean
make
make install

In the "lib" subdirectory of the BMI installation directory, run the following script, adjusting ZOID_DIR in line 3 as needed:

#!/bin/sh

ZOID_DIR=$HOME/ZeptoOS-bmi/packages/zoid
ZOID_SRCDIR=$ZOID_DIR/src
ZOID_PREBUILTDIR=$ZOID_DIR/prebuilt

if [ ! -f libbmi.a ]; then
	echo "libbmi.a not found in the current working directory!"
	exit 1
fi

if [ -f $ZOID_SRCDIR/zbmi/libzbmi_cn.a ]; then
	ar x $ZOID_SRCDIR/zbmi/libzbmi_cn.a
else
	ar x $ZOID_PREBUILTDIR/libzbmi_cn.a
fi
if [ -f $ZOID_SRCDIR/cnl/client/libzoid_cn.a ]; then
	ar x $ZOID_SRCDIR/cnl/client/libzoid_cn.a
else
	ar x $ZOID_PREBUILTDIR/libzoid_cn.a
fi

ar r libbmi.a *.o

rm *.o

exit 0

Building PVFS2 for IOFSL PVFS2 Dispatcher:

LDFLAGS="-L$ZOID_DIR/prebuilt/" LIBS="-lzbmi_cn -lzoid_cn" ./configure --prefix=$SOFT_BASE --with-zoid=$ZOID_DIR/src --disable-server --disable-karma
make clean
make
make install

libsysio (optional)

cvs -d:pserver:[email protected]:/cvsroot/libsysio login
cvs -z3 -d:pserver:[email protected]:/cvsroot/libsysio co -P libsysio
cd libsysio
./autogen.sh
./configure --prefix=$SOFT_BASE --with-statvfs=yes --with-native-driver=yes --with-incore-driver=no --with-file-handle-interface=yes --with-threads=posix --with-stdfd-dev=no --with-stddev-dev=no --with-alternate-symbols="_zfs_sysio_"
make
make install

iofsl

See GitRepository on how to obtain the source.

Building

./prepare
CC="ccache gcc" CXX="ccache g++" CFLAGS="-g -I$SOFT_BASE/include -g -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64" CXXFLAGS="-g -I$SOFT_BASE/include -g -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64" LDFLAGS="-L$SOFT_BASE/lib -L$ZOID_DIR/prebuilt" LIBS="-lzbmi_cn -lzoid_cn" ./configure --prefix=$SOFT_BASE --disable-fuse --enable-nofs --with-libsysio=/home/kota/iofsl/libsysio
make clean
make
make install

zepto + mpich2 + ROMIO-IOFSL

You may need the newer version of Subversion on Surveyor.

svn checkout https://svn.mcs.anl.gov/repos/ZeptoOS/trunk/
cd zoid-bmi
rm extra/config/mconf # something nasty
./configure
make -C comm rebuild-target
cp ./comm/DCMF/build/mpich2/dcmf-8aint/lib/* $SOFT_BASE/lib/

script

$HOME/zoid-user-script.sh

#!/bin/bash

#
# launch the iofwd server in the background
#
LOG=/home/kota/log.txt

#
# if the ION is shutting down
#
if [ "$1" == "0" ] ; then
    echo "shutdown detected..." >> $LOG
    killall -9 /home/kota/run-server.sh
    echo "shutdown completed..." >> $LOG
fi

#
# if the ION is starting up
#
if [ "$1" == "1" ] ; then
    echo "ION startup detected..." >> $LOG
    /home/kota/run-server.sh &
    echo "ION startup complete..." >> $LOG
fi

#
# exit
#
exit 0;

run-server.sh

#!/bin/bash
export LD_LIBRARY_PATH=/home/kota/soft/lib:$LD_LIBRARY_PATH

echo "[ION information]"
uname -a
echo $PATH

echo "[Launch IOFSL]"
/home/kota/iofsl/iofsl/src/iofwd/iofwd --config /home/kota/defaultconfig.cf >>/home/kota/log.txt 2>&1

defaultconfig.cf

See the ConfigurationFile page for details on how to configure the IOFSL server through the configuration file.

compile MPI-IO code

$ /soft/apps/ZeptoOS/bin/zmpicc -L$SOFT_BASE/lib -L/home/iskra/iofsl/install/bmi/lib -g -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -o simpleromio ~/simpleromio.c -lmpich.zcl -lzoidfsclient -lzoidfshints -lbmi

submit

IOR

$ qsub -A ZeptoOS -t 15 -n 1 --kernel zeptoos-bmi --mode smp --env "ZOIDFS_ION_NAME=zoid://" /home/kota/iofsl/IOR-2.10.2/src/C/IOR -a MPIIO -C -i 1 -t 1m -b 4m -e -v -v -o zoidfs:/home/kota/tmp/testfile3