This is the entire Makefile. It perhaps ought to be sent as an attachment
but my hacky mailer script wouldn't like it.
A few of the lines are wider than 80 columns, unfortunately.
-Peter
# GHC benchmarks of parsing (bytestring, basic code generation, I/O).
#
# Copyright 2007 Peter Lund , licensed under GPLv2.
#
#
# You will need the following tools:
# perl, strace, /usr/bin/time, bash, a gcc that uses shared libraries and libc
# (not dietlibc, klibc, uclibc), objdump (usually found in the binutils
# package).
#
# A benchmark run on a new platform can be split into two phases:
#
# phase1: Compiles, dices, and slices the code in various ways. If this
# completes you can be pretty sure that everything works all right.
# Performs non-timing sensitive measurements.
#
# phase2: Performs timing sensitive measurements. It is a good idea to run
# this phase on an idle machine, preferably without using X.
# For example, you can log out of X and run "telinit 1" to get to
# single-user mode. If you do so, please remember to set the correct
# path to your ghc compiler.
# Make a few bits of the makefile less noisy.
Q:=@
#########################################################
# Ask ghc to optimize and warn
GHCFLAGS= -O2 -W
# Some newer versions of gcc prefer -Wextra -Wall
GCCWARNFLAGS=-W -Wall
# Default compilers
CC=gcc
GHC=ghc
GHCPKG=ghc-pkg
#########################################################
HSPROGS=hs/byte-bs----acc \
hs/byte-bs----foldlx \
hs/byte-bs----foldrx \
hs/byte-bsl---acc \
hs/byte-xxxxx-acc-1 \
hs/byte-xxxxx-acc-2 \
hs/byte-xxxxx-foldl \
\
hs/space-bs-c8-acc-1 \
hs/space-bs-c8-count \
hs/space-bs-c8-foldlx-1 \
hs/space-bs-c8-foldlx-2 \
hs/space-bs-c8-foldrx \
hs/space-bs-c8-lenfil \
hs/space-bslc8-acc-1 \
hs/space-bslc8-acc-2 \
hs/space-bslc8-acc-3 \
hs/space-bslc8-chunk-1 \
hs/space-bslc8-chunk-2 \
hs/space-bslc8-chunk-3 \
hs/space-bslc8-chunk-4 \
hs/space-bslc8-count \
hs/space-bslc8-foldl \
hs/space-bslc8-foldlx-1 \
hs/space-bslc8-foldlx-2 \
hs/space-bslc8-foldr-1 \
hs/space-bslc8-foldr-2 \
hs/space-bslc8-lenfil-1 \
hs/space-bslc8-lenfil-2 \
hs/space-bsl---foldlx \
hs/space-xxxxx-acc-1 \
hs/space-xxxxx-acc-2 \
hs/space-xxxxx-foldl \
hs/space-xxxxx-lenfil
# RMPROGS keeps track of programs that are not always included in the tests.
# We do want 'make clean' to delete them even when they are not currently
# part of the build (they may be left over from a previous build).
# stack overflow with long4.
#HSPROGS:=$(HSPROGS) hs/byte-xxxxx-foldr-1
RMPROGS:=$(RMPROGS) hs/byte-xxxxx-foldr-1
# stack overflow with long4.
#HSPROGS:=$(HSPROGS) hs/byte-xxxxx-foldr-2
RMPROGS:=$(RMPROGS) hs/byte-xxxxx-foldr-2
# stack overflow with long4.
#HSPROGS:=$(HSPROGS) hs/space-xxxxx-foldr-1
RMPROGS:=$(RMPROGS) hs/space-xxxxx-foldr-1
# stack overflow with long4.
#HSPROGS:=$(HSPROGS) hs/space-xxxxx-foldr-2
RMPROGS:=$(RMPROGS) hs/space-xxxxx-foldr-2
HANDPROGS= hand/byte-bs----acc-a \
hand/byte-bs----acc-b \
hand/byte-bs----acc-c \
hand/byte-bs----acc-d \
\
hand/space-bs-c8-acc-1-a \
hand/space-bs-c8-acc-1-b \
hand/space-bs-c8-acc-1-c \
hand/space-bs-c8-acc-1-d \
hand/space-bs-c8-acc-1-e \
hand/space-bs-c8-acc-1-f \
hand/space-bs-c8-acc-1-g \
hand/space-bs-c8-acc-1-h \
hand/space-bs-c8-acc-1-i \
hand/space-bs-c8-acc-1-j \
hand/space-bs-c8-acc-1-k \
hand/space-bs-c8-acc-1-l \
hand/space-bs-c8-acc-1-m \
hand/space-bs-c8-acc-1-n \
hand/space-bs-c8-acc-1-o \
hand/space-bs-c8-acc-1-p \
hand/space-bs-c8-acc-1-q \
hand/space-bs-c8-acc-1-r \
hand/space-bs-c8-acc-1-s
RMPROGS:=$(RMPROGS) $(HANDPROGS)
ifeq ($(shell $(GHCPKG) list | grep bytestring),)
# ghc 6.6.1 with an old version of bytestring in 'base' but without its own
# module name
HSPROGS:=$(shell printf "%s\n" $(HSPROGS) | grep -v '.*-chunk-.*')
endif
HANDTEXT:=including hand-tweaked assembly
ifeq ($(shell $(GHC) --version | grep 6.9.20071119),)
HANDPROGS:=
HANDTEXT:=no hand-tweaked assembly
endif
ifneq ($(SUFFIX),)
HANDPROGS:=
HANDTEXT:=no hand-tweaked assembly
endif
CPROGS= c/byte-getchar c/byte-getchar-u c/byte-4k \
\
c/space-getchar c/space-getchar-u c/space-4k \
c/space-megabuf c/space-getwchar c/space-getwchar-u \
c/space-32k c/space-32k-8
#########################################################
# The benchmarks can be run in three modes. The default can be overridden from
# command line:
#
# make TESTKIND=SMOKETEST phase1
#
# just tests the test suite, as fast as possible
#TESTKINDDEFAULT=SMOKETEST
# small test
TESTKINDDEFAULT=NORMAL
# very thorough test
#TESTKINDDEFAULT=THOROUGH
TESTKIND=$(TESTKINDDEFAULT)
ifeq ($(TESTKIND),THOROUGH)
TESTFILE= testfiles/long4
TESTFILECACHE= testfiles/long3
else
ifeq ($(TESTKIND),NORMAL)
TESTFILE= testfiles/long3
TESTFILECACHE= testfiles/long2
else
ifeq ($(TESTKIND),SMOKETEST)
TESTFILE= testfiles/long2
TESTFILECACHE= testfiles/long1
endif
endif
endif
# Older versions of strace don't support the -E parameter which we use to
# set LD_PRELOAD before running the straced command (so we can get by with
# a single run of each benchmark in phase 1 instead of 2).
#
# Override with STRACE=OLD on the command line if you need to work with an
# old strace.
STRACE=NEW
#########################################################
.PHONY: XXXXFIRST testfiles core stg cmm asm dis discut prog \
time stat strace iotrace iosum mem cache doc phase1 phase2 redophase2
XXXXFIRST: help
testfiles: testfiles/long1 testfiles/long2 testfiles/long3 testfiles/long4
core: $(addsuffix .core ,$(HSPROGS) )
stg: $(addsuffix .stg ,$(HSPROGS) )
cmm: $(addsuffix .cmm ,$(HSPROGS) )
asm: $(addsuffix .s ,$(HSPROGS) $(CPROGS))
dis: $(addsuffix .dis ,$(HSPROGS) $(HANDPROGS) $(CPROGS))
discut: $(addsuffix .discut ,$(HSPROGS) $(HANDPROGS) $(CPROGS))
prog: $(HSPROGS) $(HANDPROGS) $(CPROGS)
time: prog \
testfiles \
$(addsuffix .time ,$(HSPROGS) $(HANDPROGS) $(CPROGS))
stat: $(addsuffix .stat ,$(HSPROGS) $(HANDPROGS) $(CPROGS))
strace: $(addsuffix .strace ,$(HSPROGS) $(HANDPROGS) $(CPROGS))
iotrace:$(addsuffix .iotrace,$(HSPROGS) $(HANDPROGS) $(CPROGS))
iosum: $(addsuffix .iosum ,$(HSPROGS) $(HANDPROGS) $(CPROGS))
mem: $(addsuffix .mem ,$(HSPROGS) $(HANDPROGS) $(CPROGS))
cache: $(addsuffix .cache ,$(HSPROGS) $(HANDPROGS) $(CPROGS))
doc: $(addsuffix .doc ,$(HSPROGS) $(HANDPROGS) $(CPROGS))
###
phase1: testfiles tools/eatmem prog iosum mem
printf "Done!\07" # beep
phase2: time report
printf "Done!\07" # beep
redophase2: cleartime
rm -f */*.srctimespace report.txt
$(MAKE) phase2
#########################################################
testfiles/long1:
mkdir -p testfiles
tools/genfiles.pl 10000 > "$@"
testfiles/long2:
mkdir -p testfiles
tools/genfiles.pl 100000 > "$@"
testfiles/long3:
mkdir -p testfiles
tools/genfiles.pl 1000000 > "$@"
testfiles/long4:
mkdir -p testfiles
tools/genfiles.pl 10000000 > "$@"
#########################################################
tools/eatmem: tools/eatmem.c
$(CC) $(GCCWARNFLAGS) -O2 "$<" -o "$@"
tools/pause-at-end.so: tools/pause-at-end.c
$(CC) $(GCCWARNFLAGS) -shared -ldl "$<" -o "$@"
#########################################################
docs: core stg cmm asm discut time iotrace doc sysinfo
rm -f docs
cat */*.doc sysinfo > docs
hs/%.doc: hs/%.core hs/%.stg hs/%.cmm hs/%.s hs/%.discut hs/%.time
(export F="$(basename $@)" ; \
printf "\n" ; \
printf "*********************************************\n" ; \
printf "****\n" ; \
printf "**** %s:\n" "$$F" ; \
printf "****\n" ; \
printf "*********************************************\n\n" ; \
printf "Haskell code:\n\n" ; \
cat "$$F.hs" ; \
printf -- "-------------------------------\n" ; \
printf "%s:\n" "$$F.core" ; \
cat "$$F.core" ; \
printf -- "-------------------------------\n" ; \
printf "%s:\n" "$$F.stg" ; \
cat "$$F.stg" ; \
printf -- "-------------------------------\n" ; \
printf "%s:\n" "$$F.cmm" ; \
cat "$$F.cmm" ; \
printf -- "-------------------------------\n" ; \
printf "%s:\n" "$$F.s" ; \
cat "$$F.s" ; \
printf -- "-------------------------------\n" ; \
printf "%s:\n" "$$F.discut" ; \
cat "$$F.discut" ; \
printf -- "-------------------------------\n" ; \
printf "%s:\n" "$$F.time" ; \
cat "$$F.time" ; \
printf -- "-------------------------------\n" ; \
printf "\014" ; \
) >> "$@"
hand/%.doc: hand/%.discut hand/%.time
(export F="$(basename $@)" ; \
printf "\n" ; \
printf "*********************************************\n" ; \
printf "****\n" ; \
printf "**** %s:\n" "$$F" ; \
printf "****\n" ; \
printf "*********************************************\n\n" ; \
printf "Haskell code:\n\n" ; \
export X=`echo $$F | sed -e 's/[a-z]$$//'` ; \
cat "$$X.hs" ; \
printf -- "-------------------------------\n" ; \
printf "%s (hand tweaked):\n" "$$F.s" ; \
cat "$$F.s" ; \
printf -- "-------------------------------\n" ; \
printf "%s:\n" "$$F.discut" ; \
cat "$$F.discut" ; \
printf -- "-------------------------------\n" ; \
printf "%s:\n" "$$F.time" ; \
cat "$$F.time" ; \
printf -- "-------------------------------\n" ; \
printf "\014" ; \
) >> "$@"
c/%.doc: c/%.s c/%.discut c/%.time
(export F="$(basename $@)" ; \
printf "\n" ; \
printf "*********************************************\n" ; \
printf "****\n" ; \
printf "**** %s:\n" "$$F" ; \
printf "****\n" ; \
printf "*********************************************\n\n" ; \
printf "C code:\n\n" ; \
cat "$$F.c" ; \
printf -- "-------------------------------\n" ; \
printf "%s:\n" "$$F.s" ; \
cat "$$F.s" ; \
printf -- "-------------------------------\n" ; \
printf "%s:\n" "$$F.discut" ; \
cat "$$F.discut" ; \
printf -- "-------------------------------\n" ; \
printf "%s:\n" "$$F.time" ; \
cat "$$F.time" ; \
printf -- "-------------------------------\n" ; \
printf "\014" ; \
) >> "$@"
#########################################################
.PHONY: report.txt lastreport
report: report.txt
$(Q)cat report.txt
report.txt \
$(addsuffix .srctimespace,$(HSPROGS) $(HANDPROGS) $(CPROGS)): \
iosum mem time stat platforminfo
$(Q)tools/genreport.pl $(HSPROGS) $(HANDPROGS) $(CPROGS) > report.txt
lastreport:
$(Q)cat report.txt
#########################################################
# Probably all or most of the targets (left-hand sides) in these rules should
# be mentioned in a .SECONDARY rule so make won't delete them behind our backs,
# in its infinite wisdom. This is sometimes necessary when using pattern
# rules (i.e. rules with '%' wildcards in them).
#
# For some reason, it doesn't seem to be all that necessary, although I had to
# insert a couple of those .SECONDARY things earlier to make make behave. For
# example, I had to insert this rule at some point but now things keep working
# even when it's commented out:
#
# .SECONDARY: $(HSPROGS) $(HANDPROGS) $(CPROGS)
#
hs/%: hs/%.hs
$(GHC) $(GHCFLAGS) --make -fforce-recomp "$<" -o "$@"
hand/%: hand/%.s
$(GHC) -no-hs-main "$<" -o "$@" -package bytestring
c/%: c/%.c
$(CC) $(GCCWARNFLAGS) -O2 "$<" -o "$@"
###
%.dis: %
@# Limit the disassembly for speed reasons (10x+ difference) and
@# file size reasons (20x-30x difference).
@# The stuff we are interested in comes early in the .text segment so
@# there's no reason to disassemble the entire runtime system, which
@# comes afterwards in case of hand/ and hs/ binaries.
objdump -M intel -D --stop-address=0x08060000 "$<" > "$@"
###
%.discut: %.dis
tools/cut.pl < "$<" > "$@"
###
%.core: %.hs
$(GHC) $(GHCFLAGS) -c -ddump-simpl "$<" > "$@"
###
%.stg: %.hs
$(GHC) $(GHCFLAGS) -c -ddump-stg "$<" > "$@"
###
%.cmm: %.hs
$(GHC) $(GHCFLAGS) -c -ddump-cmm "$<" > "$@"
###
%.s: %.hs
$(GHC) $(GHCFLAGS) -c -fforce-recomp -keep-s-files "$<"
.SECONDARY: $(addsuffix .s,$(CPROGS))
%.s: %.c
$(CC) $(GCCWARNFLAGS) -O2 -S $< -o $@
#########################################################
# The first run is sacrificial, except when smoketesting where there only is
# one run.
ifeq ($(TESTKIND),THOROUGH)
TIME= bash -c "time $<" < $(TESTFILE); \
bash -c "time $<" < $(TESTFILE); \
bash -c "time $<" < $(TESTFILE); \
bash -c "time $<" < $(TESTFILE); \
bash -c "time $<" < $(TESTFILE); \
bash -c "time $<" < $(TESTFILE)
NOSKIP:=
else
ifeq ($(TESTKIND),NORMAL)
TIME= bash -c "time $<" < $(TESTFILE); \
bash -c "time $<" < $(TESTFILE); \
bash -c "time $<" < $(TESTFILE); \
bash -c "time $<" < $(TESTFILE)
NOSKIP:=
else
ifeq ($(TESTKIND),SMOKETEST)
TIME= bash -c "time $<" < $(TESTFILE)
NOSKIP:= NOSKIP=1
endif
endif
endif
# In order to reduce the risk of swapping during the time test, we try to make
# sure there's twice the test file size free (and a bit).
# NEEDFREE is measured in kilobytes.
NEEDFREE=$(shell expr 22 '*' `ls -s $(TESTFILE) | cut -f1 -d' '` / 10)
%.time: % tools/eatmem $(TESTFILE)
printf "%s\n" "$<" > "$@"
tools/eatmem $(NEEDFREE)
dd if=$(TESTFILE) of=/dev/null
dd if="$<" of=/dev/null
($(TIME)) >>"$@" 2>&1
printf "%s\n\n" "-----" >> "$@"
%.stat: %.time
$(NOSKIP) tools/stat.pl < "$<" > "$@"
%.mem %.strace: % tools/pause-at-end.so $(TESTFILE)
ifeq ($(STRACE),OLD)
strace -o tmp.strace -f \
/usr/bin/time "$<" +RTS -sstderr < $(TESTFILE) > $(basename $@).mem 2>&1
LD_PRELOAD=tools/pause-at-end.so \
"$<" +RTS -sstderr < $(TESTFILE) >> $(basename $@).mem 2>&1
else
strace -o tmp.strace -ELD_PRELOAD=tools/pause-at-end.so -f \
/usr/bin/time "$<" +RTS -sstderr < $(TESTFILE) > $(basename $@).mem 2>&1
endif
tools/cutmem.pl < $(basename $@).mem > tmp
mv tmp $(basename $@).mem
tools/cutpid.pl < tmp.strace > $(basename $@).strace
rm -f tmp.strace
#%.strace: %.mem
# @echo > /dev/null
%.iotrace: %.strace
grep '^\(read\|write\|select\)' "$<" > "$@"
%.iosum: %.iotrace
tools/iosummary.pl < "$<" > "$@"
%.cache: % $(TESTFILECACHE)
valgrind --tool=cachegrind "$<" < $(TESTFILECACHE) 2> "$@"
#########################################################
.PHONY: zipdata help cleartime clean distclean
sysinfo:
hostname > sysinfo
cat /etc/*release >> sysinfo
@echo >> sysinfo
uname -a >> sysinfo
@echo >> sysinfo
cat /proc/cpuinfo >> sysinfo
$(GHC) --version >> sysinfo
echo >> sysinfo
$(CC) --version >> sysinfo
# This variable makes testing with weird /proc/cpuinfo files easier
CPUINFO=/proc/cpuinfo
platforminfo:
hostname > platforminfo
(printf 'ghc '; ($(GHC) --version | sed -ne 's/^The.*version //p')) >> platforminfo
cat $(CPUINFO) | sed -ne '/model name.*:/ { s/model name.*: //p; q}' >> platforminfo
printf "%s MHz\n" `cat $(CPUINFO) | sed -ne '/cpu MHz.*:/ { s/cpu MHz.*: //p; q}'` >> platforminfo
printf "TESTKIND=$(TESTKIND)\n" >> platforminfo
printf "SUFFIX=$(SUFFIX)\n" >> platforminfo
zipdata: time stat mem strace iotrace iosum sysinfo report.txt
rm -f ghc-measurements.tar.gz
rm -rf ghc-measurements
mkdir -p ghc-measurements
cp --parents \
$(addprefix */*, .time .stat .mem .iosum) sysinfo report.txt platforminfo \
ghc-measurements
printf "%s " "$(HSPROGS)" "$(HANDPROGS)" "$(CPROGS)" > ghc-measurements/progs
tar -zcf ghc-measurements.tar.gz ghc-measurements
rm -rf ghc-measurements
help:
@echo 'Measurements of very simple string I/O and parsing.'
@printf ' (%d benchmarks, %s)\n' `echo $(HSPROGS) $(HANDPROGS) $(CPROGS) | wc -w` "$(HANDTEXT)"
@echo ''
@echo ' phase1 -- preparation + measurements that can run in background'
@echo ' phase2 -- measurements that should run on unloaded machine'
@echo ' redophase2 -- rerun phase2'
@echo ''
@echo ' doc, [ASCII=1] report, lastreport - reports'
@echo ' zipdata -- zip up measurements (to ghc-measurements.tar.gz)'
@echo ''
@echo ' prog,core,stg,cmm,asm,dis,discut'
@echo ' -- compile, compile to core/stg/cmm/asm, disassemble, cut out main loop'
@echo ' time,stat,mem,strace,iotrace,iosum,cache'
@echo ' -- measure run-time, GHC heap + OS mem, syscalls, I/O patterns, cache'
@echo ''
@echo ' cleartime, clean, distclean -- delete measurements etc'
@echo ''
@echo ' TESTKIND=(SMOKETEST,NORMAL,THOROUGH), defaults to $(TESTKINDDEFAULT)'
@echo ' STRACE=OLD, defaults to NEW'
cleartime:
rm -f */*.time
clean:
# keep and hand/*.s !
rm -rf */*.hi */*.o *.o \
*/*.core */*.stg */*.cmm hs/*.s c/*.s */*.dis */*.discut \
*/*.hcr \
*/*.time */*.stat \
*/*.real \
*/*.strace */*.iotrace */*.iosum \
*/*.mem */*.cache cachegrind.out.* \
*/*.doc */*.srctimespace \
tmp.strace tmp \
tools/eatmem tools/pause-at-end.so \
$(HSPROGS) $(CPROGS) $(HANDPROGS) $(RMPROGS) a.out \
testfiles/ \
ghc-measurements/ \
sysinfo platforminfo docs xx.ps
distclean: clean
rm -f *~ */*~ report.txt ghc-measurements.tar.gz
