Add benchmarking CI (#399)

* Add benchmark file

* delete old benchmarks

* Add github action

* Add Project

* Apply suggestions from code review

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Missing end of line

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.github/workflows/benchmark.yml

@@ -0,0 +1,24 @@
+name: Run benchmarks
+
+on:
+  pull_request:
+
+jobs:
+  Benchmark:
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'performance critical')
+    env:
+      JULIA_DEBUG: BenchmarkCI
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: 1.6
+      - name: Install dependencies
+        run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI"'
+      - name: Run benchmarks
+        run: julia -e "using BenchmarkCI; BenchmarkCI.judge()"
+      - name: Post results
+        run: julia -e "using BenchmarkCI; BenchmarkCI.postjudge()"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -4,3 +4,4 @@
 /test/Manifest.toml
 coverage/
 .DS_store
+benchmark/Manifest.toml

benchmark/Project.toml

@@ -0,0 +1,3 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392"

benchmark/benchmarks.jl

@@ -1,23 +1,46 @@
 using BenchmarkTools
-using Random
-using Distances, LinearAlgebra
+using KernelFunctions
 
-const SUITE = BenchmarkGroup()
+N1 = 10
+N2 = 20
 
-Random.seed!(1234)
+X = rand(N1, N2)
+Xc = ColVecs(X)
+Xr = RowVecs(X)
+Xv = collect.(eachcol(X))
+Y = rand(N1, N2)
+Yc = ColVecs(Y)
+Yr = RowVecs(Y)
+Yv = collect.(eachcol(Y))
 
-dim = 50
-N1 = 1000;
-N2 = 500;
-alpha = 2.0
+# Create the general suite of benchmarks
+SUITE = BenchmarkGroup()
 
-X = rand(Float64, N1, dim)
-Y = rand(Float64, N2, dim)
+kernels = Dict(
+    "SqExponential" => SqExponentialKernel(), "Exponential" => ExponentialKernel()
+)
 
-KXY = rand(Float64, N1, N2)
-KX = rand(Float64, N1, N1)
-sKX = Symmetric(rand(Float64, N1, N1))
-kX = rand(Float64, N1)
+inputtypes = Dict("ColVecs" => (Xc, Yc), "RowVecs" => (Xr, Yr), "Vecs" => (Xv, Yv))
 
-include("kernelmatrix.jl")
-include("MLKernels.jl")
+functions = Dict(
+    "kernelmatrixX" => (kernel, X, Y) -> kernelmatrix(kernel, X),
+    "kernelmatrixXY" => (kernel, X, Y) -> kernelmatrix(kernel, X, Y),
+    "kernelmatrix_diagX" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X),
+    "kernelmatrix_diagXY" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X, Y),
+)
+
+for (kname, kernel) in kernels
+    SUITE[kname] = sk = BenchmarkGroup()
+    for (inputname, (X, Y)) in inputtypes
+        sk[inputname] = si = BenchmarkGroup()
+        for (fname, f) in functions
+            si[fname] = @benchmarkable $f($kernel, $X, $Y)
+        end
+    end
+end
+
+# Uncomment the following to run benchmark locally
+
+# tune!(SUITE)
+
+# results = run(SUITE, verbose=true)