[runtime] Integrate BenchmarkDotNet for in-tree microbenchmarks (reapply of #9833) (#11813)

5 files changed, 157 insertions, 1 deletions

diff --git a/acceptance-tests/Makefile.am b/acceptance-tests/Makefile.am
index 95ad60584c6..ef10f15c7e8 100644
--- a/acceptance-tests/Makefile.am
+++ b/acceptance-tests/Makefile.am
@@ -3,9 +3,10 @@ BENCHMARKER_PATH=$(ACCEPTANCE_TESTS_PATH)/benchmarker
 ROSLYN_PATH=$(ACCEPTANCE_TESTS_PATH)/roslyn
 CORECLR_PATH=$(ACCEPTANCE_TESTS_PATH)/coreclr
 MSTESTSUITE_PATH=$(ACCEPTANCE_TESTS_PATH)/ms-test-suite
+DEBIANSHOOTOUTMONO_PATH=$(ACCEPTANCE_TESTS_PATH)/DebianShootoutMono
 
 CLEANFILES = *.dll *.exe *.mdb
-EXTRA_DIST=README.md SUBMODULES.json versions.mk profiler-stress.mk roslyn.mk coreclr.mk ms-test-suite.mk
+EXTRA_DIST=README.md SUBMODULES.json microbench.mk versions.mk profiler-stress.mk roslyn.mk coreclr.mk ms-test-suite.mk
 
 CLASS=$(mcs_topdir)/class/lib/$(DEFAULT_PROFILE)
 TOOLS_CLASS=$(mcs_topdir)/class/lib/build
@@ -17,6 +18,7 @@ ILASM = $(TOOLS_RUNTIME) $(TOOLS_CLASS)/ilasm.exe
 XUNIT = $(RUNTIME) $(abs_top_builddir)/external/xunit-binaries/xunit.console.exe
 
 include versions.mk
+include microbench.mk
 include profiler-stress.mk
 include roslyn.mk
 include coreclr.mk
diff --git a/acceptance-tests/SUBMODULES.json b/acceptance-tests/SUBMODULES.json
index 66b35cb1d28..239bfce29d9 100644
--- a/acceptance-tests/SUBMODULES.json
+++ b/acceptance-tests/SUBMODULES.json
@@ -30,5 +30,13 @@
     "remote-branch": "origin/master", 
     "branch": "master", 
     "directory": "benchmarker"
+  },
+  {
+    "name": "DebianShootoutMono", 
+    "url": "https://github.com/alexanderkyte/DebianShootoutMono.git",
+    "rev": "3fde2ced806c1fe7eed81120a40d99474fa009f0",
+    "remote-branch": "origin/release_11_15_2018",
+    "branch": "release_11_15_2018",
+    "directory": "DebianShootoutMono"
   }
 ]
diff --git a/acceptance-tests/microbench-perf.sh.in b/acceptance-tests/microbench-perf.sh.in
new file mode 100644
index 00000000000..1e69aabc46c
--- /dev/null
+++ b/acceptance-tests/microbench-perf.sh.in
@@ -0,0 +1,17 @@
+#! /bin/sh
+r='@mono_build_root@'
+aotpattern="--aot="
+
+# if this is an aot invoke
+if [[ $@ =~ $aotpattern ]];
+then
+# just aot the code
+echo "$r/runtime/mono-wrapper $@"
+exec $r/runtime/mono-wrapper $@
+else
+# else run it under perf
+echo "$MONO_PERF_BINARY record -o $r/acceptance-tests/perf.data -v -s -g -- $r/mono/mini/mono-sgen $@"
+exec $MONO_PERF_BINARY record -o $r/acceptance-tests/perf.data -v -s -g $r/mono/mini/mono-sgen $@
+fi
+
+
diff --git a/acceptance-tests/microbench.mk b/acceptance-tests/microbench.mk
new file mode 100644
index 00000000000..44015fb0df6
--- /dev/null
+++ b/acceptance-tests/microbench.mk
@@ -0,0 +1,119 @@
+check-microbench: DebianShootoutMono.stamp
+	@$(MAKE) test-run-microbench
+
+DebianShootoutMono.stamp: 
+	@$(MAKE) validate-DebianShootoutMono RESET_VERSIONS=1
+	@$(MAKE) prepare-dlls
+	@touch $@
+
+abs_top_srcdir = $(abspath $(top_srcdir))
+TEST_EXE_PATH=$(abs_top_srcdir)/acceptance-tests/external/DebianShootoutMono/release/
+NET_4_X_RUNTIME=MONO_PATH=$(TEST_EXE_PATH):$(abs_top_srcdir)/mcs/class/lib/net_4_x $(abs_top_srcdir)/runtime/mono-wrapper
+FULL_AOT_RUNTIME=MONO_PATH=$(abs_top_srcdir)/mcs/class/lib/testing_aot_full $(abs_top_srcdir)/runtime/mono-wrapper
+
+PERF_BINARY=$(if $(MONO_PERF_BINARY),$(MONO_PERF_BINARY),perf)
+PERF_RUNTIME=$(abs_top_srcdir)/acceptance-tests/microbench-perf.sh
+
+define BenchmarkDotNetTemplate
+
+run-microbench-$(1):: DebianShootoutMono.stamp
+	MONO_BENCH_AOT_RUN="$(AOT_RUN_FLAGS)" \
+	MONO_BENCH_AOT_BUILD="$(AOT_BUILD_FLAGS)" \
+	MONO_BENCH_EXECUTABLE="$(abs_top_srcdir)/runtime/mono-wrapper" \
+	MONO_BENCH_PATH="$(abs_top_srcdir)/mcs/class/lib/$(TEST_PROFILE)" \
+	MONO_BENCH_INPUT="$(2)" \
+	$(NET_4_X_RUNTIME) \
+	$(TEST_EXE_PATH)/DebianShootoutMono.exe $(1)
+
+test-run-microbench:: run-microbench-$(1)
+
+run-microbench-debug-$(1):: DebianShootoutMono.stamp
+	echo MONO_PATH="$(abs_top_srcdir)/mcs/class/lib/$(TEST_PROFILE)" $(abs_top_srcdir)/runtime/mono-wrapper $(AOT_BUILD_FLAGS) $(TEST_EXE_PATH)/DebianShootoutMono.exe
+	MONO_BENCH_INPUT="$(2)" \
+	MONO_PATH="$(abs_top_srcdir)/mcs/class/lib/$(TEST_PROFILE)" \
+	$(abs_top_srcdir)/runtime/mono-wrapper $(AOT_RUN_FLAGS) $(TEST_EXE_PATH)/DebianShootoutMono.exe Run $(1)
+
+test-run-microbench-debug:: run-microbench-debug-$(1)
+
+if HOST_LINUX
+run-microbench-profiled-$(1):: microbench-results/$(1).perf.data 
+
+microbench-results/$(1).perf.data: DebianShootoutMono.stamp 
+	mkdir -p microbench-results
+	MONO_PERF_BINARY="$(PERF_BINARY)" \
+	MONO_BENCH_EXECUTABLE="$(PERF_RUNTIME)" \
+	MONO_BENCH_AOT_RUN="$(AOT_RUN_FLAGS)"\
+	MONO_BENCH_AOT_BUILD="$(AOT_BUILD_FLAGS)"\
+	MONO_BENCH_PATH="$(abs_top_srcdir)/mcs/class/lib/$(TEST_PROFILE)" \
+	MONO_BENCH_INPUT="$(2)" \
+	$(NET_4_X_RUNTIME) \
+	$(TEST_EXE_PATH)/DebianShootoutMono.exe $(1) $(2)
+	mv perf.data microbench-results/$(1).perf.data
+
+microbench-results/$(1).tmp.perf: microbench-results/$(1).perf.data
+	$(PERF_BINARY) script -i microbench-results/$(1).perf.data > microbench-results/$(1).tmp.perf
+
+microbench-results/$(1).perf-flame.svg: microbench-results/$(1).tmp.perf
+	cat microbench-results/$(1).tmp.perf | ./external/DebianShootoutMono/FlameGraph/stackcollapse-perf.pl > microbench-results/$(1).perf-folded
+	./external/DebianShootoutMono/FlameGraph/flamegraph.pl microbench-results/$(1).perf-folded > microbench-results/$(1).perf-flame.svg
+	rm microbench-results/$(1).tmp.perf
+	rm microbench-results/$(1).perf-folded
+
+MONO_PERF_FLAGS=--show-cpu-utilization -n --hierarchy -T $(MONO_PERF_ADDITIONAL_FLAGS)
+
+microbench-results/$(1).perf.report: microbench-results/$(1).perf.data
+	$(PERF_BINARY) report -i microbench-results/$(1).perf.data $(MONO_PERF_FLAGS) > microbench-results/$(1).perf.report 
+
+test-run-microbench-profiled:: run-microbench-profiled-$(1)
+
+test-run-microbench-publish-collect:: microbench-results/$(1).perf.data microbench-results/$(1).perf.report microbench-results/$(1).perf-flame.svg
+
+endif
+
+endef
+
+if HOST_LINUX
+
+test-run-microbench-perf-check:
+	$(PERF_BINARY) record -a -o perf.data -- echo "testing"
+	rm perf.data
+
+microbench-results/perf-data.zip:
+	zip microbench-results/perf-data.zip microbench-results/*.perf.data
+	rm microbench-results/*.perf.data
+
+perf-report: microbench-results/perf-data.zip
+
+perf-report-total: test-run-microbench-publish-collect
+	@$(MAKE) perf-report
+
+endif
+
+.PHONY: prepare-dlls
+
+if FULL_AOT_TESTS
+prepare-dlls: 
+	$(FULL_AOT_RUNTIME) $(AOT_BUILD_FLAGS) $(TEST_EXE_PATH)/*.{dll,exe}
+
+else
+
+prepare-dlls:
+
+endif
+
+FIXTURE_DIR=$(abs_top_srcdir)/acceptance-tests/external/DebianShootoutMono/fixtures
+
+$(eval $(call BenchmarkDotNetTemplate,Mandelbrot,))
+$(eval $(call BenchmarkDotNetTemplate,RegexRedux,$(FIXTURE_DIR)/regexredux-input.txt))
+$(eval $(call BenchmarkDotNetTemplate,KNucleotide,$(FIXTURE_DIR)/knucleotide-input.txt))
+
+$(eval $(call BenchmarkDotNetTemplate,BinaryTrees,))
+$(eval $(call BenchmarkDotNetTemplate,NBodyTest,))
+$(eval $(call BenchmarkDotNetTemplate,SpectralNorm,))
+$(eval $(call BenchmarkDotNetTemplate,Fannkuchredux,))
+$(eval $(call BenchmarkDotNetTemplate,Fasta,))
+
+$(eval $(call BenchmarkDotNetTemplate,RevComp,$(FIXTURE_DIR)/revcomp-input.txt)) 
+
+#$(eval $(call BenchmarkDotNetTemplate,GistBenchmark,$(MONO_BENCH_GIST_URL))) broken in BDN, bug filed
+
diff --git a/acceptance-tests/versions.mk b/acceptance-tests/versions.mk
index 0c070a0f114..9cf946568c1 100644
--- a/acceptance-tests/versions.mk
+++ b/acceptance-tests/versions.mk
@@ -7,6 +7,7 @@ $(eval $(call ValidateVersionTemplate,benchmarker,BENCHMARKER))
 $(eval $(call ValidateVersionTemplate,roslyn,ROSLYN))
 $(eval $(call ValidateVersionTemplate,coreclr,CORECLR))
 $(eval $(call ValidateVersionTemplate,ms-test-suite,MSTESTSUITE))
+$(eval $(call ValidateVersionTemplate,DebianShootoutMono,DEBIANSHOOTOUTMONO))
 
 # Bump the given submodule to the revision given by the REV make variable
 # If COMMIT is 1, commit the change
@@ -14,6 +15,7 @@ bump-benchmarker: __bump-benchmarker
 bump-roslyn: __bump-version-roslyn
 bump-coreclr: __bump-version-coreclr
 bump-ms-test-suite: __bump-version-ms-test-suite
+bump-DebianShootoutMono: __bump-version-DebianShootoutMono
 
 # Bump the given submodule to the branch given by the BRANCH/REMOTE_BRANCH make variables
 # If COMMIT is 1, commit the change
@@ -21,6 +23,7 @@ bump-branch-benchmarker: __bump-branch-benchmarker
 bump-branch-roslyn: __bump-branch-roslyn
 bump-branch-coreclr: __bump-branch-coreclr
 bump-branch-ms-test-suite: __bump-branch-ms-test-suite
+bump-branch-DebianShootoutMono: __bump-branch-DebianShootoutMono
 
 # Bump the given submodule to its current GIT version
 # If COMMIT is 1, commit the change
@@ -28,6 +31,7 @@ bump-current-benchmarker: __bump-current-benchmarker
 bump-current-roslyn: __bump-current-version-roslyn
 bump-current-coreclr: __bump-current-version-coreclr
 bump-current-ms-test-suite: __bump-current-version-ms-test-suite
+bump-current-DebianShootoutMono: __bump-current-version-DebianShootoutMono
 
 commit-bump-benchmarker:
 	$(MAKE) bump-benchmarker COMMIT=1
@@ -41,6 +45,9 @@ commit-bump-coreclr:
 commit-bump-ms-test-suite:
 	$(MAKE) bump-ms-test-suite COMMIT=1
 
+commit-bump-DebianShootoutMono:
+	$(MAKE) bump-DebianShootoutMono COMMIT=1
+
 commit-bump-current-benchmarker:
 	$(MAKE) bump-current-benchmarker COMMIT=1
 
@@ -52,3 +59,6 @@ commit-bump-current-coreclr:
 
 commit-bump-current-ms-test-suite:
 	$(MAKE) bump-current-ms-test-suite COMMIT=1
+
+commit-bump-current-DebianShootoutMono:
+	$(MAKE) bump-current-DebianShootoutMono COMMIT=1