#!/bin/bash # xtc - The eXTensible Compiler # Copyright (C) 2009-2012 New York University # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # version 2 as published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. # Run SuperC on linux files. Gets the configuration settings from # kbuild by running make. # set -x ################################################################################ ##################### Gather command-line parameters ########################### ################################################################################ if [[ $1 == "-?" ]]; then help=true; fi allArgs= timeout= while getopts :l:h:p:S:J:G:cPgDekL:wx:a:m:h:irvts:bTK:o:d:CAE opt; do case $opt in l) fileList=$OPTARG allArgs="$allArgs -l" ;; h) host=$OPTARG allArgs="$allArgs -h \"$OPTARG\"" ;; p) port=$OPTARG allArgs="$allArgs -p \"$OPTARG\"" ;; S) givenSupercArgs="$OPTARG" allArgs="$allArgs -S \"$OPTARG\"" ;; J) jvmFlags="$OPTARG" allArgs="$allArgs -J \"$OPTARG\"" ;; G) gccArgs="$OPTARG" allArgs="$allArgs -G \"$OPTARG\"" ;; c) compare=true allArgs="$allArgs -c" ;; P) checkParser=true allArgs="$allArgs -P" ;; g) gccParse=true allArgs="$allArgs -g" ;; D) dynamicAnalysis=true allArgs="$allArgs -D" ;; e) configInfo=true allArgs="$allArgs -e" ;; k) allConfigInfo=true allArgs="$allArgs -k" ;; L) configDir="$OPTARG" allArgs="$allArgs -L \"$OPTARG\"" ;; w) writeConfigDir=true allArgs="$allArgs -w" ;; x) makecrossScript="$OPTARG" allArgs="$allArgs -x \"$OPTARG\"" ;; a) arch="$OPTARG" allArgs="$allArgs -a \"$OPTARG\"" ;; m) masquerade="$OPTARG" allArgs="$allArgs -m \"$OPTARG\"" ;; i) saveIntermediate=true allArgs="$allArgs -i" ;; r) runtimes=true allArgs="$allArgs -r" ;; v) verbose=true allArgs="$allArgs -v" ;; t) test=true allArgs="$allArgs -t" ;; s) serverList=$OPTARG allArgs="$allArgs -s \"$OPTARG\"" remoteDir=$PWD ;; b) slave=true allArgs="$allArgs -b" ;; T) testSlaves=true allArgs="$allArgs -T" ;; K) timeout="timeout $OPTARG" allArgs="$allArgs -K \"$OPTARG\"" ;; o) outFile=$OPTARG allArgs="$allArgs -o \"$OPTARG\"" ;; d) outDir=$OPTARG allArgs="$allArgs -d \"$OPTARG\"" ;; C) compress=true allArgs="$allArgs -C" ;; A) configAST=true allArgs="$allArgs -A" ;; E) configE=true allArgs="$allArgs -E" ;; \?) echo "(`basename $0`)" "Invalid argument: -$OPTARG" ;; :) echo "(`basename $0`)" "-$OPTARG requires an argument." ;; esac done shift $(($OPTIND - 1)) file=$1 files=(${@}) if [[ ! -z "${verbose}" ]]; then set -x on fi ################################################################################ ############################## Show Help ####################################### ################################################################################ if [[ ! -z $help || (-z $file && -z $fileList && -z $host && -z $port && -z $serverList) ]]; then echo "USAGE:" echo " `basename $0` -?" echo " Show an extended help message." echo "" echo " `basename $0` [OPTIONS] file" echo " Run on one file." echo "" echo " `basename $0` [OPTIONS] -l files.txt" echo " Get files from files.txt." echo "" echo " `basename $0` [OPTIONS] -h host -p port" echo " Get filenames from host:port. See" echo " java superc.util.FilenameServer about usage." echo "" echo "OPTIONS:" echo " TOOLS" echo " -S \"args\" (S)uperC command-line arguments." echo " -J \"args\" Extra (J)VM arguments." echo " -G \"args\" Extra (G)CC preprocessor arguments for -c." echo "" echo " OPERATIONS" echo " -c (c)ompare SuperC output to configured file." echo " -P (P)arse the preprocessed file with SuperC." echo " -g Parse with (g)cc and get timings." echo " -D (D)ynamic analysis of collected statistics." echo " -r Show per-file (r)untimes. Depends on \"bc\"." echo " -E Configure SuperC-preprocessed source with Linux config*" echo " -A Configure SuperC-parsed (A)ST with Linux config*" echo " -e (e)xtract SuperC's kbuild settings." echo " -k Extract all (k)build configuration information." echo " -L dir Get (L)inux configuration from dir." echo " -w (w)rite configuration to -L dir." echo " -x make.cross script for the config. Defaults to \"make\"." echo " -a (a)rchitecture to get the config for." echo " -m file (m)asquerade as file to use its configuration." echo " -K seconds (K)ill after a timeout. Depends on \"timeout\"." echo " -t (t)est command-line args, but don't run SuperC." echo "" echo " * Requires configured Linux kernel, e.g., with \"make allyesconfig\"" echo "" echo " DISTRIBUTED OPERATION" echo " -s servers.txt Run on slave (s)ervers, not locally." echo " -b Run as slave, so ignore -s." echo " -T (T)est slave servers' SuperC installations." echo "" echo " OUTPUT" echo " -o out Write to out." echo " -d dir Write per-file output to dir/file." echo " -C Compress output with bzip2." echo " -i Save (i)ntermediate files." # echo " -v (v)erbose output." echo "" if [ ! -z $help ]; then echo "OUTPUT" echo "comparison_succeded filename SuperC output match configured file." echo "comparison_failed filename SuperC output did not match." echo "comparions_error filename There was a problem, should try comparing again." echo "" echo "NOTES" echo "To extract kbuild's configuration information, it must be" echo "configured first, e.g. \"make allyesconfig ARCH=x86\"." echo "Using -h will read configuration information from cached files" echo "instead of using kbuild." echo "" echo "Slave servers will append their hostname to their out files." fi exit 0 fi ################################################################################ ################# Check Environment and Command-Line Arguments ################# ################################################################################ if [ ! -z $runtimes ]; then which bc >/dev/null 2>&1 if [[ $? -eq 0 ]]; then calc="bc -lq" else calc= #empty echo "Calculator program \"bc\" not found on system, reporting start and end times instead." fi fi if [ ! -z $file ]; then if [ ! -f $file ]; then echo "\"$file\" does not exist." exit fi if [ ! -z $fileList ]; then echo "Cannot specify \"-l\" and provide a filename on the command-line." exit fi if [[ ! -z $serverList || ! -z $host || ! -z $port ]]; then echo "Cannot specify \"-s\", \"-h\", or \"-p\" and provide a filename on the command-line." exit fi fi if [ ! -z $fileList ]; then if [ ! -f $fileList ]; then echo "\"$fileList\" does not exist." exit fi if [[ ! -z $serverList || ! -z $host || ! -z $port ]]; then echo "Cannot specify \"-s\", \"-h\", or \"-p\" and provide a file list with \"-l\"." exit fi fi if [[ ! -z $host || ! -z $port ]]; then if [[ -z $host || -z $port ]]; then echo "Must specify \"-h\" and \"-p\" with \"-s\"." exit fi fi if [ ! -z $slave ]; then serverList= if [ ! -z $outFile ]; then outFile=$outFile.$HOSTNAME fi fi if [ ! -z $serverList ]; then if [ ! -f $serverList ]; then echo "\"$serverList\" does not exist." exit fi echo $allArgs servers=`cat $serverList` if [ ! -z $testSlaves ]; then for server in $servers; do echo "Checking $server:$remoteDir" ssh $server "cd $remoteDir;superc_linux.sh -c -S-printSource arch/x86/kernel/acpi/realmode/regs.c" 2>&1 | grep comparison_succeeded if [ $? -ne 0 ]; then echo $retval echo "$server failed." exit fi done fi fi if [[ ! -z $outFile && -f $outFile ]]; then echo "The output file \"$outFile\" already exists." exit fi if [ ! -z $outDir ]; then if [ ! -z $compare ]; then echo "-d has no effect with -c." exit fi fi if [[ ! -z "$runtimes" && (! -z "$outDir" || ! -z "$compare") ]]; then echo "-r has no effect with -c and -d." exit fi if [[ ! -z "$writeConfigDir" && -z "$configDir" ]]; then echo "-w only applies when using -L" exit fi if [[ ! -z "$makecrossScript" && -z "$writeConfigDir" ]]; then echo "-x only applies when using -w" exit fi if [[ ! -z "$arch" && -z "$writeConfigDir" ]]; then echo "-a only applies when using -w" exit fi if [[ ! -z $test ]]; then echo "Parameters passed tests. Run again without \"-t\"." exit fi if [[ ! -z $timeout ]]; then which timeout > /dev/null 2>&1 if [ $? -ne 0 ]; then echo "The timeout utility was not found. Can't use -K." exit fi fi if [[ ! -z $compress && -z $outDir ]]; then echo "-C can only be used with -d" exit fi ################################################################################ ############################## Setup Files ##################################### ################################################################################ filesToProcess= length=0 if [ ! -z $file ]; then filesToProcess[0]=$file length=1 # if [ ! -z $files ]; then # filesToProcess=(${files[@]}) # length=${#files[@]} elif [ ! -z $fileList ]; then for file in `cat $fileList`; do filesToProcess[length]=$file length=$length+1 done fi which mktemp >/dev/null 2>&1 if [ $? -ne 0 ]; then echo "superc_linux.sh requires mktemp to create temp files" exit 1 fi tempfilebase=$(mktemp superXXXXXXXX) || exit tempfiles="$tempfilebase" header_args=$tempfilebase.headers || exit config_args=$tempfilebase.configs || exit superc_out= superc_out_E= if [ -z $saveIntermediate ]; then tempfiles="$tempfiles $header_args $config_args" fi if [ ! -z $compare ]; then superc_out=$tempfilebase.super.c || exit superc_out_E=$tempfilebase.super.E || exit gcc_E=$tempfilebase.gcc.E || exit superc_diff=$tempfilebase.diff || exit if [ -z $saveIntermediate ]; then tempfiles="$tempfiles $superc_out $superc_out_E $gcc_E $header_args $superc_diff" fi elif [ ! -z "$checkParser" ]; then gcc_E=$tempfilebase.gcc.E || exit superc_out=$tempfilebase.super.c || exit if [ -z $saveIntermediate ]; then tempfiles="$tempfiles $gcc_E $superc_out" fi fi if [ ! -z "$tempfiles" ]; then trap "rm -f -- $tempfiles" EXIT fi ################################################################################ ############################### Run SuperC ##################################### ################################################################################ if [ ! -z $outFile ]; then # Redirect all output from the script to standard out and # $outFile. # exec 2>&1 | tee > $outFile exec > $outFile 2>&1 fi if [ -z "$servers" ]; then i=0 while :; do # Get next file. if [ -z $host ]; then if [[ $i -eq $length ]]; then exit; fi file=${filesToProcess[$i]} i=$i+1 else file=`java superc.util.FilenameService -client $host $port` errcode=$? if [[ $errcode -eq 3 ]]; then echo "No more files." exit elif [[ $errcode -eq 4 ]]; then echo "No server available." exit elif [[ $errcode -ne 0 || -z $file ]]; then echo "Unknown file server error." exit fi fi echo "Processing $file" if [ ! -z "$dynamicAnalysis" ]; then dynamic_analysis.sh $file else # Extract configuration settings from kbuild. if [ ! -z "$masquerade" ]; then config_file=$masquerade obj_file=${masquerade%.*}.o else config_file=$file obj_file=${file%.*}.o fi if [[ ! -z "$configE" ]]; then supercArgs="$givenSupercArgs -E" elif [[ ! -z "$configAST" ]]; then supercArgs="$givenSupercArgs -configFile .config -configureExceptions \"$(java superc.util.GCCShunt --emit-exceptions $(superc_linux.sh -k "$config_file" | grep ^configs_all) none)\"" else supercArgs="$givenSupercArgs" fi if [ ! -z "$configDir" ]; then cacheConfigArgs=$configDir/$config_file.configs_all cacheSuperCArgs=$configDir/$config_file.configs_superc if [ -z "$writeConfigDir" ]; then config_args=$cacheConfigArgs header_args=$cacheSuperCArgs if [[ -f $config_args && -f $header_args ]]; then getConfigResult=0 else echo "excluded $file no configuration from $configDir" getConfigResult=1 fi fi fi if [[ -f $cacheConfigArgs && -f $cacheSuperCArgs && ! -z "$writeConfigDir" ]]; then echo "excluded $file already written" getConfigResult=0; elif [[ -z "$configDir" || ! -z "$writeConfigDir" ]]; then # Can't override CC since it is used to emit compiler # information if [[ -z "$makecrossScript" ]]; then makeScript="make" else makeScript="$makecrossScript" fi which $makeScript if [[ -z "$arch" ]]; then archParam="" else archParam="ARCH=$arch" fi $makeScript $archParam CHECK="java superc.util.GCCShunt --shunt-filename $config_file --shunt-superc $header_args --shunt-kbuild --shunt-config $config_args" C=2 $obj_file getConfigResult=$? # if [ $getConfigResult -ne 0 ]; then # # Try overriding CC instead of CHECK, since the file # # may not be compiling. # make CC="java superc.util.GCCShunt --shunt-filename $config_file --shunt-superc $header_args --shunt-kbuild --shunt-config $config_args" C=2 $obj_file # getConfigResult=$? # fi # make -i -f $JAVA_DEV_ROOT/scripts/Makefile.kcu ARCH=x86 \ # CC="java superc.util.GCCShunt --shunt-filename $config_file --shunt-superc $header_args --shunt-kbuild --shunt-config $config_args" \ # LD="echo ld" ar="echo ar" $obj_file # getConfigResult=$? cat $config_args cat $header_args echo $getConfigResult if [[ ! -e $config_args && ! -e $header_args ]]; then echo "excluded $file no configuration from make" else getConfigResult=0 fi fi if [ $getConfigResult -eq 0 ]; then if [ ! -z "$writeConfigDir" ]; then mkdir -p `dirname $cacheConfigArgs` `dirname $cacheSuperCArgs` if [[ -f $config_args && -f $header_args ]]; then cp $config_args $cacheConfigArgs cp $header_args $cacheSuperCArgs fi elif [ ! -z "$configInfo" ]; then echo "configs_superc $file`cat $header_args`" elif [ ! -z "$allConfigInfo" ]; then echo "configs_all $file`cat $config_args`" elif [ ! -z "$checkParser" ]; then gcc -E `cat $config_args` $gccArgs $file > $gcc_E 2>/dev/null java -ea $JAVA_ARGS $jvmFlags superc.SuperC -enc-in UTF-8 -silent -showAccepts $supercArgs $gcc_E 2>&1 | tee $superc_out cat $superc_out | grep ACCEPT if [ $? -ne 0 ]; then echo "grammar_failed $file" else echo "grammar_succeeded $file" fi if [ ! -z $saveIntermediate ]; then echo $gcc_E echo $superc_out echo $header_args echo $config_args fi elif [ ! -z "$gccParse" ]; then gcc -ftime-report -fsyntax-only `cat $config_args` $file else # bash is simply a terrible programming language. # that's why this mess below is needed. it # handles quoted command-line arguments, # preventing bash from separating them into # tokens. cmd=(java -ea $JAVA_ARGS $jvmFlags superc.SuperC -enc-in UTF-8 -silent) saveifs=$IFS IFS=' ' for cmd_i in $(grep -Eo '"[^"]*"|[^" ]*' <<< "$supercArgs"); do cmd_i="${cmd_i%\"}" cmd_i="${cmd_i#\"}" cmd+=("$cmd_i") done IFS=$saveifs cmd+=(`cat $header_args` -include $JAVA_DEV_ROOT/data/cpp/nonbooleans.h $file) # # debugging output # for (( cmd_i=0; cmd_i<${#cmd[@]}; cmd_i++ )); do # echo ${cmd[cmd_i]} # done # exit # cmd="java -ea $JAVA_ARGS $jvmFlags superc.SuperC -enc-in UTF-8 -silent $supercArgs `cat $header_args` -include $JAVA_DEV_ROOT/data/cpp/nonbooleans.h $file" if [ -z $compare ]; then if [ -z $outDir ]; then before=`date +%s.%N` $timeout "${cmd[@]}" if [ $? -eq 124 ]; then echo "killed $file" fi after=`date +%s.%N` if [ ! -z "$runtimes" ]; then if [[ ! -z $calc ]]; then time=`echo "$after - $before" | $calc` echo "performance $file $time" else # If there's no calculator program, # output raw start and end times. echo "performance_raw $file $before $after" fi fi else outDirCombined=$outDir/`dirname $file` mkdir -p $outDirCombined outFile=$outDir/$file $timeout "${cmd[@]}" > $outFile.part 2>&1 if [ $? -eq 124 ]; then echo "killed $file" | tee -a $outFile.part fi mv $outFile.part $outFile if [ ! -z $compress ]; then bzip2 $outFile fi fi else $timeout "${cmd[@]}" | grep -v "^Resizing node table" > $superc_out if [ $? -eq 124 ]; then echo "killed $file" | tee -a $superc_out fi gcc -E `cat $config_args` $gccArgs $superc_out > $superc_out_E gcc -E `cat $config_args` $gccArgs $file > $gcc_E 2>/dev/null java superc.util.cdiff -s $superc_out_E $gcc_E > $superc_diff 2>&1 retval=$? cat $superc_diff if [ $retval -eq 0 ]; then echo "comparison_succeeded $file" elif [ $retval -eq 1 ]; then echo "comparison_failed $file" else echo "comparison_error $file" fi if [ ! -z $saveIntermediate ]; then echo $superc_diff echo $gcc_E echo $superc_out echo $superc_out_E echo $header_args echo $config_args fi fi fi fi fi done else # Run SuperC on remote servers, not locally. for server in $servers; do echo "Starting slave $server:$remoteDir" ssh $server "source .profile; cd $remoteDir; superc_linux.sh -b $allArgs" & done fi