#!/usr/bin/env python # # download and build a standalone epics environment for the xspress3 # # to use this script: # mkdir /home/xspress3/epics # cd /home/xspress3/epics # wget https://raw.githubusercontent.com/epics-modules/xspress3/master/build_xspress3.py # python build_xspress3.py all # See INSTALL_Standalone.md for details. ####### # Note: centos7 defaults with python2, so this needs to work # with both Py2 and Py3 from __future__ import print_function import os import sys import multiprocessing as mp import subprocess as sp import shutil import time from argparse import ArgumentParser modules = {'base': 'base-7.0.7.tar.gz', 'asyn': 'asyn-R4-44-2.tar.gz', 'ipac': 'ipac-2.16.tar.gz', 'areaDetector': 'areaDetector-R3-12-1.tar.gz', 'busy': 'busy-R1-7-4.tar.gz', 'alive': 'alive-R1-4-1.tar.gz', 'std': 'std-R3-6-4.tar.gz', 'autosave': 'autosave-R5-11.tar.gz', 'iocStats': 'iocStats-3.2.0.tar.gz', 'sscan': 'sscan-R2-11-5.tar.gz', 'calc': 'calc-R3-7-5.tar.gz', 'sncseq': 'seq-2.2.6.tar.gz', 'xspress3': 'xspress3-2-8.tar.gz'} ad_modules = {'ADSupport': 'adSupport-R1-10.tar.gz', 'ADCore': 'adCore-R3-12-1.tar.gz'} other_sources = ('procServ-2.8.0.tar.gz', 'bin_softioc.tar.gz', 'medm_ext.tar.gz') SOURCES_URL = 'https://millenia.cars.aps.anl.gov/software/xspress3/sources' LARCH_URL = 'https://millenia.cars.aps.anl.gov/xraylarch/downloads/' GETLARCH = 'GetLarch.sh' ###################################################################### required_tools = {'re2c': '/bin/re2c', 'rpcgen': '/bin/rpcgen', 'readline-devel': '/usr/lib64/libreadline.so', 'hdf5-devel': '/usr/include/hdf5.h', 'libtiff-devel': '/usr/include/tiff.h', 'bzip2-devel': '/usr/include/bzlib.h', 'libxml2-devel': '/usr/lib64/libxml2.so', 'GraphicsMagick-devel ': '/usr/lib64/libGraphicsMagick.so'} recommended_tools = {'telnet': '/bin/telnet', 'motif-devel': '/usr/lib64/libXm.so'} HELP_MESSAGE = '''build_xspress3.py: build Epics Xspress3 module options: -h, --help shows this message and exit -n, --nelem number of detector elements to configure for [4] arguments: extract download (if needed) and extract source files only configure configure sources only build compile sources only (must be configured first) xrfapp install and configure XRF display app all extract, configure, and compile sources, install xrfapp clean run 'make clean' for all modules realclean remove all module folders created by 'build' distclean do realclean and remove source packages examples: To build everything for 4 element detector: > python build_xspress3.py -n 4 all ''' COPY_DMFILES = ''' find . -name "*.adl" | sed \'s|^|cp |g\' | sed \'s|$| adls/.|g\' > tmp_copy_dmfiles find . -name "*.ui" | sed \'s|^|cp |g\' | sed \'s|$| uis/.|g\' >> tmp_copy_dmfiles find . -name "*.opi" | sed \'s|^|cp |g\' | sed \'s|$| opis/.|g\' >> tmp_copy_dmfiles sh tmp_copy_dmfiles rm tmp_copy_dmfiles ''' HOME_DIR = os.environ.get('HOME', '/home/xspress3') START_IOC = '''#!/bin/bash TOPDIR={topdir:s} IOCNAME=ioc_{nelem:d}Channel SCRIPT=st.cmd APPNAME=$TOPDIR/xspress3/iocs/xspress3IOC/bin/linux-x86_64/xspress3App IOCDIR=$TOPDIR/xspress3/iocs/xspress3IOC/iocBoot/$IOCNAME source $TOPDIR/bin/bash_profile.sh cd $IOCDIR $APPNAME $SCRIPT ''' RUN_MEDM = '''#!/bin/bash {topdir:s}/bin/medm -x -macro "P=XSP3_{nelem:d}Chan" {topdir:s}/adls/xspress3_{nelem:d}chan.adl ''' BUILD_PROCSERV = ''' cd procServ-2.8.0 && sh ./configure && make -j8 && cp procServ ../bin/. && cd .. ''' BUILD_MEDM = ''' cd extensions/src/medm && make -j8 && cd ../../.. && cp extensions/bin/linux-x86_64/medm bin/. ''' AD_RELEASE_LOCAL = ''' ADSUPPORT=$(AREA_DETECTOR)/ADSupport -include $(TOP)/configure/RELEASE.local.$(EPICS_HOST_ARCH) ''' AD_RELEASE_LIBS_LOCAL = ''' SUPPORT={topdir:s} EPICS_BASE={topdir:s}/base ASYN=$(SUPPORT)/asyn AREA_DETECTOR=$(SUPPORT)/areaDetector ADSUPPORT=$(AREA_DETECTOR)/ADSupport ADCORE=$(AREA_DETECTOR)/ADCore -include $(AREA_DETECTOR)/configure/RELEASE_LIBS.local.$(EPICS_HOST_ARCH) ''' AD_RELEASE_PRODS_LOCAL = ''' SUPPORT={topdir:s} EPICS_BASE={topdir:s}/base ASYN=$(SUPPORT)/asyn AREA_DETECTOR=$(SUPPORT)/areaDetector ADSUPPORT=$(AREA_DETECTOR)/ADSupport ADCORE=$(AREA_DETECTOR)/ADCore AUTOSAVE=$(SUPPORT)/autosave BUSY=$(SUPPORT)/busy CALC=$(SUPPORT)/calc SNCSEQ=$(SUPPORT)/sncseq SSCAN=$(SUPPORT)/sscan DEVIOCSTATS=$(SUPPORT)/iocStats -include $(AREA_DETECTOR)/configure/RELEASE_PRODS.local.$(EPICS_HOST_ARCH) ''' AD_CONFIG_LOCAL = ''' WITH_BOOST = NO BOOST_EXTERNAL = YES WITH_PVA = YES WITH_QSRV = YES WITH_BLOSC = YES BLOSC_EXTERNAL = NO WITH_BITSHUFFLE = YES BITSHUFFLE_EXTERNAL = NO WITH_GRAPHICSMAGICK = YES GRAPHICSMAGICK_EXTERNAL = NO GRAPHICSMAGICK_PREFIX_SYMBOLS = YES WITH_HDF5 = YES HDF5_EXTERNAL = NO WITH_JPEG = YES JPEG_EXTERNAL = NO WITH_NETCDF = YES NETCDF_EXTERNAL = NO WITH_NEXUS = YES NEXUS_EXTERNAL = NO WITH_OPENCV = NO OPENCV_EXTERNAL = YES WITH_SZIP = YES SZIP_EXTERNAL = NO WITH_TIFF = YES TIFF_EXTERNAL = NO XML2_EXTERNAL = NO WITH_ZLIB = YES ZLIB_EXTERNAL = NO ARAVIS_INCLUDE = /usr/local/include/aravis-0.8 GLIB_INCLUDE = /usr/include/glib-2.0 /usr/lib64/glib-2.0/include glib-2.0_DIR = /usr/lib64 ''' PROFILE = '''#!/bin/bash # startup script to use epics environment for xspress3 export EPICS_HOST_ARCH=linux-x86_64 export EPICS_CA_MAX_ARRAY_BYTES=768000 export EPICS_BASE={topdir:s}/base export EPICS_EXTENSIONS={topdir:s}/extensions export EPICS_DISPLAY_PATH={topdir:s}/adls # turn this off to make sure PVs can be seen by other # machines on your network unset EPICS_CAS_INTF_ADDR_LIST # setting EPICS Address list, there are 2 options: # a) use automatic address lookup. # works well for machines with 1 NIC (xpsress3 minis?) unset EPICS_CA_ADDR_LIST export EPICS_CA_AUTO_ADDR_LIST=YES # b) turn off automatic address lookup, explicitly set address # list to the broadcast address (or localhost). This gives # fewer warning messages for machines with multiple NICs, # such as non-mini xspress3 units. # export EPICS_CA_AUTO_ADDR_LIST=NO # export EPICS_CA_ADDR_LIST=localhost # export EPICS_CA_ADDR_LIST={broadcast:s} # include bin folders in PATH PATH={homedir:s}/xraylarch/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin export PATH=$PATH:$EPICS_BASE/bin/linux-x86_64:$EPICS_EXTENSIONS/bin/linux-x86_64:{topdir:s}/bin # include bin folders in LD_LIBRARY_PATH LD_LIBRARY_PATH=/usr/local/lib export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$EPICS_BASE/lib/linux-x86_64:$EPICS_EXTENSIONS/lib/linux-x86_64 ''' XRFAPP_SCRIPT = '''#!{homedir:s}/xraylarch/bin/python # Epics XRF Display App from larch.epics import EpicsXRFApp # Note: 'det_type' determines the layout of the buttons for the # individual detector elements. Known layouts include: # ME-4 Hitachi ME-4 # ME-7 Hitachi ME-7 # SXD-7 Canberra SXD-7 # other all other detector type (display 4 per row) app = EpicsXRFApp(nmca={nelem:d}, det_type='other', environ_file='{topdir:s}/bin/Xspress3.env', prefix='XSP3_{nelem:d}Chan:', ioc_type='Xspress3') app.MainLoop() ''' def remove_link_or_dir(name): if os.path.islink(name): os.unlink(name) elif os.path.isdir(name): shutil.rmtree(name) def write_bash_profile(nelem=4): # grep /sbin/ifconfig output for broadcast broadcast = '' for line in sp.check_output(['/usr/sbin/ifconfig']).splitlines(): if (sys.version_info.major == 3): line = line.decode('utf-8') if 'inet' in line and 'broadcast' in line: words = [i.strip() for i in line.split('broadcast')] broadcast = words[-1] break with open ('bin/bash_profile.sh', 'w') as fh: fh.write(PROFILE.format(topdir=os.getcwd(), homedir=HOME_DIR, broadcast=broadcast)) with open ('bin/run_xspress3.sh', 'w') as fh: fh.write(START_IOC.format(topdir=os.getcwd(), nelem=nelem)) with open ('bin/run_medm.sh', 'w') as fh: fh.write(RUN_MEDM.format(topdir=os.getcwd(), nelem=nelem)) os.chmod('bin/bash_profile.sh', 493) # mod 755 os.chmod('bin/run_xspress3.sh', 493) # mod 755 os.chmod('bin/run_medm.sh', 493) # mod 755 print('# wrote bash_profile.sh') def create_bindir(): tarball = 'bin_softioc.tar.gz' tball = os.path.abspath(os.path.join('sources', tarball)) cwd = os.path.abspath(os.getcwd()) if not os.path.exists(tball): os.chdir('sources') print("# fetch %s" % ('%s/%s' % (SOURCES_URL, tarball))) o = sp.call(['/bin/wget', '%s/%s' % (SOURCES_URL, tarball)]) os.chdir(cwd) if not os.path.exists('bin'): os.mkdir('bin') if not os.path.exists(tball): print("# download failed! for %s" % tball) else: tarcmd = ['tar', 'xvzf', tball] o = sp.check_output(tarcmd).splitlines() def unpack_tarball(tarball, shortname, sourcedir): tball = os.path.join(sourcedir, tarball) cwd = os.path.abspath(os.getcwd()) if not os.path.exists(sourcedir): os.mkdir(sourcedir) if not os.path.exists(tball): os.chdir(sourcedir) print("# fetch %s" % ('%s/%s' % (SOURCES_URL, tarball))) o = sp.call(['/bin/wget', '%s/%s' % (SOURCES_URL, tarball)]) os.chdir(cwd) if not os.path.exists(tball): print("# download failed! for %s" % tball) else: tarcmd = ['tar', 'xvzf', tball] o = sp.check_output(tarcmd).splitlines() first_line = o[0] if (sys.version_info.major == 3): first_line = first_line.decode('utf-8') outdir = first_line.split('/')[0] if shortname is not None: if os.path.exists(shortname): remove_link_or_dir(shortname) os.symlink(outdir, shortname) print("# downloaded and unpacked source for %s" % outdir) def extract_sources(): print('# extracting sources') try: os.mkdir('areaDetector') except: pass workers = [] if sys.version_info.major == 2: for key, value in modules.items(): unpack_tarball(value, key, 'sources') for tarball in other_sources: unpack_tarball(tarball, None, 'sources') else: with mp.Pool(8) as pool: for key, value in modules.items(): w = pool.apply_async(unpack_tarball, (value, key, 'sources')) workers.append(w) for tarball in other_sources: w = pool.apply_async(unpack_tarball, (tarball, None, 'sources')) workers.append(w) [w.wait() for w in workers] os.chdir('areaDetector') for key, value in ad_modules.items(): unpack_tarball(value, key, os.path.join('..', 'sources')) os.chdir('..') create_bindir() def configure_release(): subs = {} TOP = os.getcwd() subs['EPICS_BASE'] = '%s/base' % TOP subs['SUPPORT'] = TOP epics_mods = list(modules.keys()) for mod in epics_mods: subs[mod.upper()] = '%s/%s' % (TOP, mod) for mod in epics_mods: release = os.path.join(TOP, mod, 'configure', 'RELEASE') with open(release, 'r') as fh: text = fh.readlines() savefile = release + '.backup' if not os.path.exists(savefile): shutil.copy(release, savefile) out = [] if mod in ('asyn', ): for key in ('EPICS_BASE', 'SUPPORT'): # out.append(f'{key}={subs[key]}') out.append('%s=%s' % (key, subs[key])) for line in text: line = line[:-1] for key, val in subs.items(): if line.startswith(key): line = '%s=%s' % (key, val) out.append(line) out.append('') with open(release, 'w') as fh: fh.write('\n'.join(out)) # fix asyn rpc.h include if needed if not os.path.exists('/usr/include/rpc/rpc.h'): asyn_conffile = os.path.join(TOP, 'asyn', 'configure', 'CONFIG_SITE') shutil.copy(asyn_conffile, asyn_conffile + '.backup') buff = [] for t in open(asyn_conffile, 'r').readlines(): t = t[:-1] if 'TIRPC' in t: t = 'TIRPC=YES' buff.append(t) buff.append('') with open(asyn_conffile, 'w') as fh: fh.write('\n'.join(buff)) # write areaDetector configs ad_configs = {'RELEASE.local': AD_RELEASE_LOCAL, 'RELEASE_LIBS.local': AD_RELEASE_LIBS_LOCAL, 'RELEASE_PRODS.local': AD_RELEASE_PRODS_LOCAL, 'CONFIG_SITE.local': AD_CONFIG_LOCAL} for fname, config in ad_configs.items(): fullname = os.path.join(TOP, 'areaDetector', 'configure', fname) with open(fullname, 'w') as fh: fh.write(config.format(topdir=TOP)) def create_buildscript(nelem=4): write_bash_profile(nelem=nelem) cwd = os.path.abspath(os.getcwd()) script = ['#!/bin/bash', '# build epics sources script', 'source ./bin/bash_profile.sh', 'ROOT=`pwd`'] bline = 'cd {dirname:s} && make -j8 install && cd ..' build_mods = ('base', 'sncseq', 'asyn', 'ipac', 'alive', 'std', 'autosave', 'iocStats', 'busy', 'sscan', 'calc', 'areaDetector', 'xspress3') for mod in build_mods: dirname = os.path.join(cwd, mod) script.append(bline.format(dirname=mod)) script.append('# build medm') script.append(BUILD_MEDM) script.append('# build procServ utility') script.append(BUILD_PROCSERV) script.append('# copy display files') script.append('mkdir -p adls uis opis') script.append(COPY_DMFILES) script.append('') with open('do_build.sh', 'w') as fh: fh.write('\n'.join(script)) time.sleep(0.25) print("# wrote do_build.sh. Now ready to build with 'sh do_build.sh'") def install_xrfapp(nelem): cwd = os.path.abspath(os.getcwd()) if not os.path.exists(os.path.join(HOME_DIR, 'xraylarch')): os.chdir('sources') o = sp.call(['/bin/wget', '%s%s' % (LARCH_URL, GETLARCH)]) o = sp.call(['sh', GETLARCH]) os.chdir(cwd) with open('bin/run_xrfapp.py', 'w') as fh: fh.write(XRFAPP_SCRIPT.format(topdir=cwd, homedir=HOME_DIR, nelem=nelem)) os.chmod('bin/run_xrfapp.py', 493) # mod 755 # environ file buff = [] template = 'XSP3_{nelem:d}Chan:C{elem:d}SCA:{sca:d}:Value_RBV Chan{elem:d} {label:s}' for sca, label in enumerate(('ClockTicks', 'ResetTicks', 'ResetCounts', 'AllEvents', 'AllGood', 'Window1', 'Window2', 'Pileup', 'EventWidth', 'DeadTimeFactor', 'DeadTimePercent')): for elem in range(1, nelem+1): buff.append(template.format(nelem=nelem, elem=elem, sca=sca, label=label)) buff.append('') with open('bin/Xspress3.env', 'w') as fh: fh.write('\n'.join(buff)) # make desktop shortcut shortcut_command = ['pyshortcut', '%s/support/bin/run_xrfapp.py'%(HOME_DIR), '-n', 'Xspress3_Detector', '--icon', '%s/support/xspress3/documentation/source/_static/ptable.ico'%(HOME_DIR)] sp.call(shortcut_command) def check_dependencies(): missing_reqs = [] for name, exe in required_tools.items(): if not os.path.exists(exe): missing_reqs.append(name) if len(missing_reqs) > 0: print("There are missing dependencies. Install with `sudo yum install`:") print(" ".join(missing_reqs)) sys.exit(0) missing_tools = [] for name, exe in recommended_tools.items(): if not os.path.exists(exe): missing_tools.append(name) return missing_tools def build(nelem=4): missing_tools = check_dependencies() if not os.path.exists('do_build.sh'): create_buildscript(nelem=nelem) configure_release() o = sp.call(['sh', 'do_build.sh']) if len(missing_tools) > 0: print("There are missing useful tools. Install with `sudo yum install`:") print(" ".join(missing_tools)) def clean(): cwd = os.path.abspath(os.getcwd()) for slink in modules: os.chdir(slink) o = sp.call(['make', 'clean']) os.chdir(cwd) def realclean(with_source_kits=False): if os.path.exists('do_build.sh'): os.unlink('do_build.sh') for slink, tarball in modules.items(): remove_link_or_dir(slink) sdir = tarball.replace('.tar.gz', '') remove_link_or_dir(sdir) for dname in ('do_build.sh', 'adls', 'uis', 'opis', 'procServ', 'procServ-2.8.0', 'extensions'): remove_link_or_dir(dname) if with_source_kits: remove_link_or_dir('sources') def distclean(): realclean(with_source_kits=True) if __name__ == '__main__': parser = ArgumentParser(prog='xspress3_build', description='build xspress3 epics support') # parser.add_argument('-h', '--help', dest='help') parser.add_argument('-n', '--nelem', dest='nelem', type=int, default=4) parser.add_argument('options', nargs='*') args = parser.parse_args() nelem = args.nelem if len(args.options) == 0: print(HELP_MESSAGE) else: cmd = args.options.pop(0) if cmd == 'extract': extract_sources() elif cmd == 'configure': create_buildscript(nelem=nelem) configure_release() elif cmd == 'xrfapp': install_xrfapp(nelem) elif cmd == 'clean': clean() elif cmd == 'realclean': realclean() elif cmd == 'build': build() elif cmd == 'all': xrfapp_proc = mp.Process(target=install_xrfapp, args=(nelem,)) xrfapp_proc.start() extract_sources() create_buildscript(nelem=nelem) configure_release() build() xrfapp_proc.join()