Python and distutils on the BG/P

Python for the front end

Building python for the front end is pretty straightforward. Here is an example build for Python 2.6.1:

$ ../Python-2.6.1/configure --prefix=/home/norris/soft --enable-shared -disable-ipv6 --without-threads CC=gcc LDFLAGS= CXX=g++ LINKCC=gcc
$ make
$ make install

Unfortunately, a regular Python distribution is pretty much useless for cross-compiling, in particular the distutils module contains almost no support for either IBM compilers or cross-compilation where the Python used to build the code is different from the runtime Python.

I added a module to Lib/distutils in the Python source tree, called mpixlccompiler.py, which started out as a copy of unixcompiler.py:

"""distutils.mpixlccompiler

Contains the MPIXLCCompiler class, a subclass of CCompiler that handles
the IBM cross-compilation on Blue Gene/P.
  * macros defined with -Dname[=value]
  * macros undefined with -Uname
  * include search directories specified with -Idir
  * libraries specified with -lllib
  * library search directories specified with -Ldir
  * compile handled by 'cc' (or similar) executable with -c option:
    compiles .c to .o
  * link static library handled by 'ar' command (possibly with 'ranlib')
  * link shared library handled by 'mpixlc -G -nostaticlink'
"""

__revision__ = "$Id: mpicxxcompiler.py 65012 2009-02-25 13:24:06Z boyana.norris $"

import os, sys
from types import StringType, NoneType

from distutils import sysconfig
from distutils.dep_util import newer
from distutils.ccompiler import \
     CCompiler, gen_preprocess_options, gen_lib_options
from distutils.errors import \
     DistutilsExecError, CompileError, LibError, LinkError
from distutils import log

class MPIXLCCompiler(CCompiler):

    compiler_type = 'bgp'
    #compiler_class['mpixlc'] = ('mpixlccompiler', 'MPIXLCCompiler', "Blue Gene/P Cross Compiler")


    # These are used by CCompiler in two places: the constructor sets
    # instance attributes 'preprocessor', 'compiler', etc. from them, and
    # 'set_executable()' allows any of these to be set.  The defaults here
    # are pretty generic; they will probably have to be set by an outsider
    # (eg. using information discovered by the sysconfig about building
    # Python extensions).
    executables = {'preprocessor' : ["gcc -E"],
                   'compiler'     : ["mpixlc_r","-qsuppress=1506-1108","-qalias=noansi","-qrtti=all"],
                   'compiler_so'  : ["mpixlc_r","-qsuppress=1506-1108","-qalias=noansi","-qrtti=all","-qpic","-DPIC"],
                   'compiler_cxx' : ["mpixlcxx_r"],
                   'linker_so'    : ["mpixlc_r"],
                   'linker_exe'   : ["mpixlc_r"],
                   'linker_flags' : ["--shared","-dy","-qnostaticlink"],
                   'archiver'     : ["ar", "-cr"],
                   'ranlib'       : None,
                  }

    # Needed for the filename generation methods provided by the base
    # class, CCompiler.  NB. whoever instantiates/uses a particular
    # UnixCCompiler instance should set 'shared_lib_ext' -- we set a
    # reasonable common default here, but it's not necessarily used on all
    # Unices!

    src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"]
    obj_extension = ".o"
    static_lib_extension = ".a"
    shared_lib_extension = ".so"
    static_lib_format = shared_lib_format = "lib%s%s"
    if sys.platform == "cygwin":
        exe_extension = ".exe"

    def preprocess(self, source,
                   output_file=None, macros=None, include_dirs=None,
                   extra_preargs=None, extra_postargs=None):
        if '/bgsys/drivers/ppcfloor/gnu-linux/include/python2.5' not in include_dirs:
            include_dirs.insert(0,'/bgsys/drivers/ppcfloor/gnu-linux/include/python2.5')
        ignore, macros, include_dirs_ignore = \
            self._fix_compile_args(None, macros, include_dirs)
        pp_opts = gen_preprocess_options(macros, include_dirs)
        pp_args = self.preprocessor + pp_opts
        if output_file:
            pp_args.extend(['-o', output_file])
        if extra_preargs:
            pp_args[:0] = extra_preargs
        if extra_postargs:
            pp_args.extend(extra_postargs)
        pp_args.append(source)

        # We need to preprocess: either we're being forced to, or we're
        # generating output to stdout, or there's a target output file and
        # the source file is newer than the target (or the target doesn't
        # exist).
        if self.force or output_file is None or newer(source, output_file):
            if output_file:
                self.mkpath(os.path.dirname(output_file))
            try:
                self.spawn(pp_args)
            except DistutilsExecError, msg:
                raise CompileError, msg

    def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts):
        compiler_so = self.compiler_so
        try:
            self.spawn(compiler_so + cc_args + [src, '-o', obj] +
                       extra_postargs)
        except DistutilsExecError, msg:
            raise CompileError, msg

    def create_static_jib(self, objects, output_libname,
                          output_dir=None, debug=0, target_lang=None):
        objects, output_dir = self._fix_object_args(objects, output_dir)

        output_filename = \
            self.library_filename(output_libname, output_dir=output_dir)

        if self._need_link(objects, output_filename):
            self.mkpath(os.path.dirname(output_filename))
            self.spawn(self.archiver +
                       [output_filename] +
                       objects + self.objects)

            # Not many Unices required ranlib anymore -- SunOS 4.x is, I
            # think the only major Unix that does.  Maybe we need some
            # platform intelligence here to skip ranlib if it's not
            # needed -- or maybe Python's configure script took care of
            # it for us, hence the check for leading colon.
            if self.ranlib:
                try:
                    self.spawn(self.ranlib + [output_filename])
                except DistutilsExecError, msg:
                    raise LibError, msg
        else:
            log.debug("skipping %s (up-to-date)", output_filename)

    def link(self, target_desc, objects,
             output_filename, output_dir=None, libraries=None,
             library_dirs=None, runtime_library_dirs=None,
             export_symbols=None, debug=0, extra_preargs=None,
             extra_postargs=None, build_temp=None, target_lang=None):

        objects, output_dir = self._fix_object_args(objects, output_dir)

        # These are the "standard" locations of the compute-node libraries on the bgp:
        if '/bgsys/drivers/ppcfloor/gnu-linux/lib' not in self.library_dirs:
            self.library_dirs.insert(0,'/bgsys/drivers/ppcfloor/gnu-linux/lib')
        if '/bgsys/drivers/ppcfloor/gnu-linux/lib' not in self.runtime_library_dirs:
            self.runtime_library_dirs.insert(0,'/bgsys/drivers/ppcfloor/gnu-linux/lib')
        libraries, library_dirs, runtime_library_dirs = \
           self._fix_lib_args(libraries, library_dirs, runtime_library_dirs)
        if 'python2.5' not in self.libraries:
            self.libraries.insert(0,'python2.5')

        lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs,
                                   libraries)

        for opt in lib_opts[:]:
            if opt.startswith('-lpython'): lib_opts.remove(opt)

        if type(output_dir) not in (StringType, NoneType):
            raise TypeError, "'output_dir' must be a string or None"
        if output_dir is not None:
            output_filename = os.path.join(output_dir, output_filename)

        if self._need_link(objects, output_filename):
            ld_args = (objects + self.linker_flags +  self.objects +
                       lib_opts + ['-o', output_filename])
            if debug:
                ld_args[:0] = ['-g']
            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend(extra_postargs)
            self.mkpath(os.path.dirname(output_filename))
            try:
                if target_desc == CCompiler.EXECUTABLE:
                    linker = self.linker_exe[:]
                else:
                    linker = self.linker_so[:]
                if target_lang == "c++" and self.compiler_cxx:
                    # skip over environment variable settings if /usr/bin/env
                    # is used to set up the linker's environment.
                    # This is needed on OSX. Note: this assumes that the
                    # normal and C++ compiler have the same environment
                    # settings.
                    i = 0
                    if os.path.basename(linker[0]) == "env":
                        i = 1
                        while '=' in linker[i]:
                            i = i + 1

                    linker[i] = self.compiler_cxx[i]

                #print linker, ld_args
                self.spawn(linker + ld_args)
            except DistutilsExecError, msg:
                raise LinkError, msg
        else:
            log.debug("skipping %s (up-to-date)", output_filename)

    # -- Miscellaneous methods -----------------------------------------
    # These are all used by the 'gen_lib_options() function, in
    # ccompiler.py.

    def library_dir_option(self, dir):
        return "-L" + dir

    def runtime_library_dir_option(self, dir):
        return "-R" + dir

    def library_option(self, lib):
        return "-l" + lib

    def find_library_file(self, dirs, lib, debug=0):
        shared_f = self.library_filename(lib, lib_type='shared')
        static_f = self.library_filename(lib, lib_type='static')

        for dir in dirs:
            shared = os.path.join(dir, shared_f)
            static = os.path.join(dir, static_f)
            # We're second-guessing the linker here, with not much hard
            # data to go on: GCC seems to prefer the shared library, so I'm
            # assuming that *all* Unix C compilers do.  And of course I'm
            # ignoring even GCC's "-static" option.  So sue me.
            if os.path.exists(shared):
                return shared
            elif os.path.exists(static):
                return static

        # Oops, didn't find it in *any* of 'dirs'
        return None

Then, I had to modify ccompiler.py slightly to make distutils aware of the new compiler (it's a bit dumb about finding modules dynamically). Basically I added the last entry to the compiler_class dictionary:

compiler_class = { 'unix':    ('unixccompiler', 'UnixCCompiler',
                               "standard UNIX-style compiler"),
                   'msvc':    ('msvccompiler', 'MSVCCompiler',
                               "Microsoft Visual C++"),
                   'cygwin':  ('cygwinccompiler', 'CygwinCCompiler',
                               "Cygwin port of GNU C Compiler for Win32"),
                   'mingw32': ('cygwinccompiler', 'Mingw32CCompiler',
                               "Mingw32 port of GNU C Compiler for Win32"),
                   'bcpp':    ('bcppcompiler', 'BCPPCompiler',
                               "Borland C++ Compiler"),
                   'mwerks':  ('mwerkscompiler', 'MWerksCompiler',
                               "MetroWerks CodeWarrior"),
                   'emx':     ('emxccompiler', 'EMXCCompiler',
                               "EMX port of GNU C Compiler for OS/2"),
                   'mpixlc':  ('mpixlccompiler', 'MPIXLCCompiler', 
                               "Blue Gene/P Cross Compiler")
                 }

The Python patched in that manner can be installed and used to cross-compile using the IBM compilers and the compute-node version of Python (which doesn't have to be the same version as the one used for building, e.g., I built with Python 2.6.1 while the compute-node version is 2.5).