This is the entire Makefile. It perhaps ought to be sent as an attachment but my hacky mailer script wouldn't like it. A few of the lines are wider than 80 columns, unfortunately. -Peter # GHC benchmarks of parsing (bytestring, basic code generation, I/O). # # Copyright 2007 Peter Lund <firefly@vax64.dk>, licensed under GPLv2. # # # You will need the following tools: # perl, strace, /usr/bin/time, bash, a gcc that uses shared libraries and libc # (not dietlibc, klibc, uclibc), objdump (usually found in the binutils # package). # # A benchmark run on a new platform can be split into two phases: # # phase1: Compiles, dices, and slices the code in various ways. If this # completes you can be pretty sure that everything works all right. # Performs non-timing sensitive measurements. # # phase2: Performs timing sensitive measurements. It is a good idea to run # this phase on an idle machine, preferably without using X. # For example, you can log out of X and run "telinit 1" to get to # single-user mode. If you do so, please remember to set the correct # path to your ghc compiler. # Make a few bits of the makefile less noisy. Q:=@ ######################################################### # Ask ghc to optimize and warn GHCFLAGS= -O2 -W # Some newer versions of gcc prefer -Wextra -Wall GCCWARNFLAGS=-W -Wall # Default compilers CC=gcc GHC=ghc GHCPKG=ghc-pkg ######################################################### HSPROGS=hs/byte-bs----acc \ hs/byte-bs----foldlx \ hs/byte-bs----foldrx \ hs/byte-bsl---acc \ hs/byte-xxxxx-acc-1 \ hs/byte-xxxxx-acc-2 \ hs/byte-xxxxx-foldl \ \ hs/space-bs-c8-acc-1 \ hs/space-bs-c8-count \ hs/space-bs-c8-foldlx-1 \ hs/space-bs-c8-foldlx-2 \ hs/space-bs-c8-foldrx \ hs/space-bs-c8-lenfil \ hs/space-bslc8-acc-1 \ hs/space-bslc8-acc-2 \ hs/space-bslc8-acc-3 \ hs/space-bslc8-chunk-1 \ hs/space-bslc8-chunk-2 \ hs/space-bslc8-chunk-3 \ hs/space-bslc8-chunk-4 \ hs/space-bslc8-count \ hs/space-bslc8-foldl \ hs/space-bslc8-foldlx-1 \ hs/space-bslc8-foldlx-2 \ hs/space-bslc8-foldr-1 \ hs/space-bslc8-foldr-2 \ hs/space-bslc8-lenfil-1 \ hs/space-bslc8-lenfil-2 \ hs/space-bsl---foldlx \ hs/space-xxxxx-acc-1 \ hs/space-xxxxx-acc-2 \ hs/space-xxxxx-foldl \ hs/space-xxxxx-lenfil # RMPROGS keeps track of programs that are not always included in the tests. # We do want 'make clean' to delete them even when they are not currently # part of the build (they may be left over from a previous build). # stack overflow with long4. #HSPROGS:=$(HSPROGS) hs/byte-xxxxx-foldr-1 RMPROGS:=$(RMPROGS) hs/byte-xxxxx-foldr-1 # stack overflow with long4. #HSPROGS:=$(HSPROGS) hs/byte-xxxxx-foldr-2 RMPROGS:=$(RMPROGS) hs/byte-xxxxx-foldr-2 # stack overflow with long4. #HSPROGS:=$(HSPROGS) hs/space-xxxxx-foldr-1 RMPROGS:=$(RMPROGS) hs/space-xxxxx-foldr-1 # stack overflow with long4. #HSPROGS:=$(HSPROGS) hs/space-xxxxx-foldr-2 RMPROGS:=$(RMPROGS) hs/space-xxxxx-foldr-2 HANDPROGS= hand/byte-bs----acc-a \ hand/byte-bs----acc-b \ hand/byte-bs----acc-c \ hand/byte-bs----acc-d \ \ hand/space-bs-c8-acc-1-a \ hand/space-bs-c8-acc-1-b \ hand/space-bs-c8-acc-1-c \ hand/space-bs-c8-acc-1-d \ hand/space-bs-c8-acc-1-e \ hand/space-bs-c8-acc-1-f \ hand/space-bs-c8-acc-1-g \ hand/space-bs-c8-acc-1-h \ hand/space-bs-c8-acc-1-i \ hand/space-bs-c8-acc-1-j \ hand/space-bs-c8-acc-1-k \ hand/space-bs-c8-acc-1-l \ hand/space-bs-c8-acc-1-m \ hand/space-bs-c8-acc-1-n \ hand/space-bs-c8-acc-1-o \ hand/space-bs-c8-acc-1-p \ hand/space-bs-c8-acc-1-q \ hand/space-bs-c8-acc-1-r \ hand/space-bs-c8-acc-1-s RMPROGS:=$(RMPROGS) $(HANDPROGS) ifeq ($(shell $(GHCPKG) list | grep bytestring),) # ghc 6.6.1 with an old version of bytestring in 'base' but without its own # module name HSPROGS:=$(shell printf "%s\n" $(HSPROGS) | grep -v '.*-chunk-.*') endif HANDTEXT:=including hand-tweaked assembly ifeq ($(shell $(GHC) --version | grep 6.9.20071119),) HANDPROGS:= HANDTEXT:=no hand-tweaked assembly endif ifneq ($(SUFFIX),) HANDPROGS:= HANDTEXT:=no hand-tweaked assembly endif CPROGS= c/byte-getchar c/byte-getchar-u c/byte-4k \ \ c/space-getchar c/space-getchar-u c/space-4k \ c/space-megabuf c/space-getwchar c/space-getwchar-u \ c/space-32k c/space-32k-8 ######################################################### # The benchmarks can be run in three modes. The default can be overridden from # command line: # # make TESTKIND=SMOKETEST phase1 # # just tests the test suite, as fast as possible #TESTKINDDEFAULT=SMOKETEST # small test TESTKINDDEFAULT=NORMAL # very thorough test #TESTKINDDEFAULT=THOROUGH TESTKIND=$(TESTKINDDEFAULT) ifeq ($(TESTKIND),THOROUGH) TESTFILE= testfiles/long4 TESTFILECACHE= testfiles/long3 else ifeq ($(TESTKIND),NORMAL) TESTFILE= testfiles/long3 TESTFILECACHE= testfiles/long2 else ifeq ($(TESTKIND),SMOKETEST) TESTFILE= testfiles/long2 TESTFILECACHE= testfiles/long1 endif endif endif # Older versions of strace don't support the -E parameter which we use to # set LD_PRELOAD before running the straced command (so we can get by with # a single run of each benchmark in phase 1 instead of 2). # # Override with STRACE=OLD on the command line if you need to work with an # old strace. STRACE=NEW ######################################################### .PHONY: XXXXFIRST testfiles core stg cmm asm dis discut prog \ time stat strace iotrace iosum mem cache doc phase1 phase2 redophase2 XXXXFIRST: help testfiles: testfiles/long1 testfiles/long2 testfiles/long3 testfiles/long4 core: $(addsuffix .core ,$(HSPROGS) ) stg: $(addsuffix .stg ,$(HSPROGS) ) cmm: $(addsuffix .cmm ,$(HSPROGS) ) asm: $(addsuffix .s ,$(HSPROGS) $(CPROGS)) dis: $(addsuffix .dis ,$(HSPROGS) $(HANDPROGS) $(CPROGS)) discut: $(addsuffix .discut ,$(HSPROGS) $(HANDPROGS) $(CPROGS)) prog: $(HSPROGS) $(HANDPROGS) $(CPROGS) time: prog \ testfiles \ $(addsuffix .time ,$(HSPROGS) $(HANDPROGS) $(CPROGS)) stat: $(addsuffix .stat ,$(HSPROGS) $(HANDPROGS) $(CPROGS)) strace: $(addsuffix .strace ,$(HSPROGS) $(HANDPROGS) $(CPROGS)) iotrace:$(addsuffix .iotrace,$(HSPROGS) $(HANDPROGS) $(CPROGS)) iosum: $(addsuffix .iosum ,$(HSPROGS) $(HANDPROGS) $(CPROGS)) mem: $(addsuffix .mem ,$(HSPROGS) $(HANDPROGS) $(CPROGS)) cache: $(addsuffix .cache ,$(HSPROGS) $(HANDPROGS) $(CPROGS)) doc: $(addsuffix .doc ,$(HSPROGS) $(HANDPROGS) $(CPROGS)) ### phase1: testfiles tools/eatmem prog iosum mem printf "Done!\07" # beep phase2: time report printf "Done!\07" # beep redophase2: cleartime rm -f */*.srctimespace report.txt $(MAKE) phase2 ######################################################### testfiles/long1: mkdir -p testfiles tools/genfiles.pl 10000 > "$@" testfiles/long2: mkdir -p testfiles tools/genfiles.pl 100000 > "$@" testfiles/long3: mkdir -p testfiles tools/genfiles.pl 1000000 > "$@" testfiles/long4: mkdir -p testfiles tools/genfiles.pl 10000000 > "$@" ######################################################### tools/eatmem: tools/eatmem.c $(CC) $(GCCWARNFLAGS) -O2 "$<" -o "$@" tools/pause-at-end.so: tools/pause-at-end.c $(CC) $(GCCWARNFLAGS) -shared -ldl "$<" -o "$@" ######################################################### docs: core stg cmm asm discut time iotrace doc sysinfo rm -f docs cat */*.doc sysinfo > docs hs/%.doc: hs/%.core hs/%.stg hs/%.cmm hs/%.s hs/%.discut hs/%.time (export F="$(basename $@)" ; \ printf "\n" ; \ printf "*********************************************\n" ; \ printf "****\n" ; \ printf "**** %s:\n" "$$F" ; \ printf "****\n" ; \ printf "*********************************************\n\n" ; \ printf "Haskell code:\n\n" ; \ cat "$$F.hs" ; \ printf -- "-------------------------------\n" ; \ printf "%s:\n" "$$F.core" ; \ cat "$$F.core" ; \ printf -- "-------------------------------\n" ; \ printf "%s:\n" "$$F.stg" ; \ cat "$$F.stg" ; \ printf -- "-------------------------------\n" ; \ printf "%s:\n" "$$F.cmm" ; \ cat "$$F.cmm" ; \ printf -- "-------------------------------\n" ; \ printf "%s:\n" "$$F.s" ; \ cat "$$F.s" ; \ printf -- "-------------------------------\n" ; \ printf "%s:\n" "$$F.discut" ; \ cat "$$F.discut" ; \ printf -- "-------------------------------\n" ; \ printf "%s:\n" "$$F.time" ; \ cat "$$F.time" ; \ printf -- "-------------------------------\n" ; \ printf "\014" ; \ ) >> "$@" hand/%.doc: hand/%.discut hand/%.time (export F="$(basename $@)" ; \ printf "\n" ; \ printf "*********************************************\n" ; \ printf "****\n" ; \ printf "**** %s:\n" "$$F" ; \ printf "****\n" ; \ printf "*********************************************\n\n" ; \ printf "Haskell code:\n\n" ; \ export X=`echo $$F | sed -e 's/[a-z]$$//'` ; \ cat "$$X.hs" ; \ printf -- "-------------------------------\n" ; \ printf "%s (hand tweaked):\n" "$$F.s" ; \ cat "$$F.s" ; \ printf -- "-------------------------------\n" ; \ printf "%s:\n" "$$F.discut" ; \ cat "$$F.discut" ; \ printf -- "-------------------------------\n" ; \ printf "%s:\n" "$$F.time" ; \ cat "$$F.time" ; \ printf -- "-------------------------------\n" ; \ printf "\014" ; \ ) >> "$@" c/%.doc: c/%.s c/%.discut c/%.time (export F="$(basename $@)" ; \ printf "\n" ; \ printf "*********************************************\n" ; \ printf "****\n" ; \ printf "**** %s:\n" "$$F" ; \ printf "****\n" ; \ printf "*********************************************\n\n" ; \ printf "C code:\n\n" ; \ cat "$$F.c" ; \ printf -- "-------------------------------\n" ; \ printf "%s:\n" "$$F.s" ; \ cat "$$F.s" ; \ printf -- "-------------------------------\n" ; \ printf "%s:\n" "$$F.discut" ; \ cat "$$F.discut" ; \ printf -- "-------------------------------\n" ; \ printf "%s:\n" "$$F.time" ; \ cat "$$F.time" ; \ printf -- "-------------------------------\n" ; \ printf "\014" ; \ ) >> "$@" ######################################################### .PHONY: report.txt lastreport report: report.txt $(Q)cat report.txt report.txt \ $(addsuffix .srctimespace,$(HSPROGS) $(HANDPROGS) $(CPROGS)): \ iosum mem time stat platforminfo $(Q)tools/genreport.pl $(HSPROGS) $(HANDPROGS) $(CPROGS) > report.txt lastreport: $(Q)cat report.txt ######################################################### # Probably all or most of the targets (left-hand sides) in these rules should # be mentioned in a .SECONDARY rule so make won't delete them behind our backs, # in its infinite wisdom. This is sometimes necessary when using pattern # rules (i.e. rules with '%' wildcards in them). # # For some reason, it doesn't seem to be all that necessary, although I had to # insert a couple of those .SECONDARY things earlier to make make behave. For # example, I had to insert this rule at some point but now things keep working # even when it's commented out: # # .SECONDARY: $(HSPROGS) $(HANDPROGS) $(CPROGS) # hs/%: hs/%.hs $(GHC) $(GHCFLAGS) --make -fforce-recomp "$<" -o "$@" hand/%: hand/%.s $(GHC) -no-hs-main "$<" -o "$@" -package bytestring c/%: c/%.c $(CC) $(GCCWARNFLAGS) -O2 "$<" -o "$@" ### %.dis: % @# Limit the disassembly for speed reasons (10x+ difference) and @# file size reasons (20x-30x difference). @# The stuff we are interested in comes early in the .text segment so @# there's no reason to disassemble the entire runtime system, which @# comes afterwards in case of hand/ and hs/ binaries. objdump -M intel -D --stop-address=0x08060000 "$<" > "$@" ### %.discut: %.dis tools/cut.pl < "$<" > "$@" ### %.core: %.hs $(GHC) $(GHCFLAGS) -c -ddump-simpl "$<" > "$@" ### %.stg: %.hs $(GHC) $(GHCFLAGS) -c -ddump-stg "$<" > "$@" ### %.cmm: %.hs $(GHC) $(GHCFLAGS) -c -ddump-cmm "$<" > "$@" ### %.s: %.hs $(GHC) $(GHCFLAGS) -c -fforce-recomp -keep-s-files "$<" .SECONDARY: $(addsuffix .s,$(CPROGS)) %.s: %.c $(CC) $(GCCWARNFLAGS) -O2 -S $< -o $@ ######################################################### # The first run is sacrificial, except when smoketesting where there only is # one run. ifeq ($(TESTKIND),THOROUGH) TIME= bash -c "time $<" < $(TESTFILE); \ bash -c "time $<" < $(TESTFILE); \ bash -c "time $<" < $(TESTFILE); \ bash -c "time $<" < $(TESTFILE); \ bash -c "time $<" < $(TESTFILE); \ bash -c "time $<" < $(TESTFILE) NOSKIP:= else ifeq ($(TESTKIND),NORMAL) TIME= bash -c "time $<" < $(TESTFILE); \ bash -c "time $<" < $(TESTFILE); \ bash -c "time $<" < $(TESTFILE); \ bash -c "time $<" < $(TESTFILE) NOSKIP:= else ifeq ($(TESTKIND),SMOKETEST) TIME= bash -c "time $<" < $(TESTFILE) NOSKIP:= NOSKIP=1 endif endif endif # In order to reduce the risk of swapping during the time test, we try to make # sure there's twice the test file size free (and a bit). # NEEDFREE is measured in kilobytes. NEEDFREE=$(shell expr 22 '*' `ls -s $(TESTFILE) | cut -f1 -d' '` / 10) %.time: % tools/eatmem $(TESTFILE) printf "%s\n" "$<" > "$@" tools/eatmem $(NEEDFREE) dd if=$(TESTFILE) of=/dev/null dd if="$<" of=/dev/null ($(TIME)) >>"$@" 2>&1 printf "%s\n\n" "-----" >> "$@" %.stat: %.time $(NOSKIP) tools/stat.pl < "$<" > "$@" %.mem %.strace: % tools/pause-at-end.so $(TESTFILE) ifeq ($(STRACE),OLD) strace -o tmp.strace -f \ /usr/bin/time "$<" +RTS -sstderr < $(TESTFILE) > $(basename $@).mem 2>&1 LD_PRELOAD=tools/pause-at-end.so \ "$<" +RTS -sstderr < $(TESTFILE) >> $(basename $@).mem 2>&1 else strace -o tmp.strace -ELD_PRELOAD=tools/pause-at-end.so -f \ /usr/bin/time "$<" +RTS -sstderr < $(TESTFILE) > $(basename $@).mem 2>&1 endif tools/cutmem.pl < $(basename $@).mem > tmp mv tmp $(basename $@).mem tools/cutpid.pl < tmp.strace > $(basename $@).strace rm -f tmp.strace #%.strace: %.mem # @echo > /dev/null %.iotrace: %.strace grep '^\(read\|write\|select\)' "$<" > "$@" %.iosum: %.iotrace tools/iosummary.pl < "$<" > "$@" %.cache: % $(TESTFILECACHE) valgrind --tool=cachegrind "$<" < $(TESTFILECACHE) 2> "$@" ######################################################### .PHONY: zipdata help cleartime clean distclean sysinfo: hostname > sysinfo cat /etc/*release >> sysinfo @echo >> sysinfo uname -a >> sysinfo @echo >> sysinfo cat /proc/cpuinfo >> sysinfo $(GHC) --version >> sysinfo echo >> sysinfo $(CC) --version >> sysinfo # This variable makes testing with weird /proc/cpuinfo files easier CPUINFO=/proc/cpuinfo platforminfo: hostname > platforminfo (printf 'ghc '; ($(GHC) --version | sed -ne 's/^The.*version //p')) >> platforminfo cat $(CPUINFO) | sed -ne '/model name.*:/ { s/model name.*: //p; q}' >> platforminfo printf "%s MHz\n" `cat $(CPUINFO) | sed -ne '/cpu MHz.*:/ { s/cpu MHz.*: //p; q}'` >> platforminfo printf "TESTKIND=$(TESTKIND)\n" >> platforminfo printf "SUFFIX=$(SUFFIX)\n" >> platforminfo zipdata: time stat mem strace iotrace iosum sysinfo report.txt rm -f ghc-measurements.tar.gz rm -rf ghc-measurements mkdir -p ghc-measurements cp --parents \ $(addprefix */*, .time .stat .mem .iosum) sysinfo report.txt platforminfo \ ghc-measurements printf "%s " "$(HSPROGS)" "$(HANDPROGS)" "$(CPROGS)" > ghc-measurements/progs tar -zcf ghc-measurements.tar.gz ghc-measurements rm -rf ghc-measurements help: @echo 'Measurements of very simple string I/O and parsing.' @printf ' (%d benchmarks, %s)\n' `echo $(HSPROGS) $(HANDPROGS) $(CPROGS) | wc -w` "$(HANDTEXT)" @echo '' @echo ' phase1 -- preparation + measurements that can run in background' @echo ' phase2 -- measurements that should run on unloaded machine' @echo ' redophase2 -- rerun phase2' @echo '' @echo ' doc, [ASCII=1] report, lastreport - reports' @echo ' zipdata -- zip up measurements (to ghc-measurements.tar.gz)' @echo '' @echo ' prog,core,stg,cmm,asm,dis,discut' @echo ' -- compile, compile to core/stg/cmm/asm, disassemble, cut out main loop' @echo ' time,stat,mem,strace,iotrace,iosum,cache' @echo ' -- measure run-time, GHC heap + OS mem, syscalls, I/O patterns, cache' @echo '' @echo ' cleartime, clean, distclean -- delete measurements etc' @echo '' @echo ' TESTKIND=(SMOKETEST,NORMAL,THOROUGH), defaults to $(TESTKINDDEFAULT)' @echo ' STRACE=OLD, defaults to NEW' cleartime: rm -f */*.time clean: # keep and hand/*.s ! rm -rf */*.hi */*.o *.o \ */*.core */*.stg */*.cmm hs/*.s c/*.s */*.dis */*.discut \ */*.hcr \ */*.time */*.stat \ */*.real \ */*.strace */*.iotrace */*.iosum \ */*.mem */*.cache cachegrind.out.* \ */*.doc */*.srctimespace \ tmp.strace tmp \ tools/eatmem tools/pause-at-end.so \ $(HSPROGS) $(CPROGS) $(HANDPROGS) $(RMPROGS) a.out \ testfiles/ \ ghc-measurements/ \ sysinfo platforminfo docs xx.ps distclean: clean rm -f *~ */*~ report.txt ghc-measurements.tar.gz