Benchmarks

Here we visualize benchmarks included in Proffer’s test suite.

These benchmarks were run on a typical laptop, and use both mock and real data. Due to differences in the graph structure of mock data, different runtime profiles can be observed in several areas.

!date --iso

2024-12-06

Setup

import numpy as np
import pandas as pd
import polars as pl
import seaborn as sns

from matplotlib import pyplot as plt
from matplotlib_venn import venn2, venn3

import os
from pathlib import Path
from urllib.parse import urlparse

import fsspec
import json
import toml

DARK_MODE = True

sns.set_theme(context="talk", style="ticks")#, palette="pastel")
if DARK_MODE:
  plt.style.use(style="dark_background")

seer_colors = ["#22c6c9", "#702f8a", "#faaa8d", "#bfb8af", "#0032a0", "#00a9ce", "#e4d5d3", "#000000"]
_pal = sns.color_palette(seer_colors)
sns.set_palette(_pal)

Load Benchmarks

with open("../.benchmarks/Linux-CPython-3.10-64bit/0008_2024-10-15-subsume-fix.json", "r") as f:
    bench_data = json.load(f)

print(toml.dumps(bench_data["machine_info"]))

node = "sjust-pclaptop-seer"
processor = "x86_64"
machine = "x86_64"
python_compiler = "GCC 11.2.0"
python_implementation = "CPython"
python_implementation_version = "3.10.14"
python_version = "3.10.14"
python_build = [ "main", "May  6 2024 19:42:50",]
release = "6.9.3-76060903-generic"
system = "Linux"

[cpu]
python_version = "3.10.14.final.0 (64 bit)"
cpuinfo_version = [ 9, 0, 0,]
cpuinfo_version_string = "9.0.0"
arch = "X86_64"
bits = 64
count = 32
arch_string_raw = "x86_64"
vendor_id_raw = "GenuineIntel"
brand_raw = "Intel(R) Core(TM) i9-14900HX"
hz_advertised_friendly = "5.0369 GHz"
hz_actual_friendly = "5.0369 GHz"
hz_advertised = [ 5036922000, 0,]
hz_actual = [ 5036922000, 0,]
stepping = 1
model = 183
family = 6
flags = [ "3dnowprefetch", "abm", "acpi", "adx", "aes", "aperfmperf", "apic", "arat", "arch_capabilities", "arch_lbr", "arch_perfmon", "art", "avx", "avx2", "avx_vnni", "bmi1", "bmi2", "bts", "clflush", "clflushopt", "clwb", "cmov", "constant_tsc", "cpuid", "cpuid_fault", "cx16", "cx8", "de", "ds_cpl", "dtes64", "dtherm", "dts", "epb", "ept", "ept_ad", "erms", "est", "f16c", "flexpriority", "flush_l1d", "fma", "fpu", "fsgsbase", "fsrm", "fxsr", "gfni", "hfi", "ht", "hwp", "hwp_act_window", "hwp_epp", "hwp_notify", "hwp_pkg_req", "ibpb", "ibrs", "ibrs_enhanced", "ibt", "ida", "intel_pt", "invpcid", "lahf_lm", "lm", "mca", "mce", "md_clear", "mmx", "monitor", "movbe", "movdir64b", "movdiri", "msr", "mtrr", "nonstop_tsc", "nopl", "nx", "ospke", "osxsave", "pae", "pat", "pbe", "pcid", "pclmulqdq", "pdcm", "pdpe1gb", "pebs", "pge", "pku", "pln", "pni", "popcnt", "pse", "pse36", "pts", "rdpid", "rdrand", "rdrnd", "rdseed", "rdtscp", "rep_good", "sdbg", "sep", "serialize", "sha", "sha_ni", "smap", "smep", "split_lock_detect", "ss", "ssbd", "sse", "sse2", "sse4_1", "sse4_2", "ssse3", "stibp", "syscall", "tm", "tm2", "tpr_shadow", "tsc", "tsc_adjust", "tsc_deadline_timer", "tsc_known_freq", "tscdeadline", "umip", "user_shstk", "vaes", "vme", "vmx", "vnmi", "vpclmulqdq", "vpid", "waitpkg", "x2apic", "xgetbv1", "xsave", "xsavec", "xsaveopt", "xsaves", "xtopology", "xtpr",]
l3_cache_size = 37748736
l2_cache_size = 33554432
l1_data_cache_size = 917504
l1_instruction_cache_size = "1.3 MiB"
l2_cache_line_size = 4096
l2_cache_associativity = 8

_bench_df = pl.from_records(bench_data["benchmarks"])

bench_df = (
    _bench_df.select(
        pl.col("name")
            .str.extract(r"^(.*?)(?:\[.+?\])?$")
            .alias("name"),
        "params",
        "stats",
    )
    .unnest(
        "params",
        "stats",
    )
    .with_columns(
        inference_method=pl.col("inference_method")
            .str.extract(r"\[<function (\w+).*"),
    )
)

bench_df.get_column("name").unique(maintain_order=True).to_list()

['test_benchmark_edges_to_peptides',
 'test_benchmark_build_sparse_graph_mock',
 'test_benchmark_build_sparse_graph_real',
 'test_benchmark_get_connected_components_mock',
 'test_benchmark_get_connected_components_real',
 'test_benchmark_infer_sparse_graph_mock',
 'test_benchmark_infer_sparse_graph_real',
 'test_benchmark_compute_proteotypicity_mock',
 'test_benchmark_compute_proteotypicity_real',
 'test_benchmark_infer_end_to_end_real']

Visualize

# Define basic plotting function

from datetime import timedelta
import re

from humanize import precisedelta
from matplotlib.ticker import FuncFormatter

def plot_benchmark(name, param_cols, bench_df=bench_df, method="cat", tight_layout=True, **kwargs):
    if isinstance(param_cols, str):
        param_cols = [param_cols]

    param_cols = list(param_cols)

    g = getattr(sns, method + "plot")(
        data=bench_df
            .filter(
                pl.col("name") == name,
            )
            .to_pandas(),
        x=param_cols.pop(0),
        y="mean",
        **dict(zip(["hue", "style", "col", "row"], param_cols)),
        **dict(
            dict(
                kind="bar",
            ),
            **kwargs,
        )
    ).set_ylabels("")

    for ax in g.axes.flat:
        ax.yaxis.set_major_formatter(FuncFormatter(
            lambda v, pos: re.sub(
                r"\w+\b",
                lambda unit: "us" if unit.group(0).startswith("microsecond")
                        else "ms" if unit.group(0).startswith("millisecond")
                        else "s" if unit.group(0).startswith("second")
                        else "m" if unit.group(0).startswith("minute")
                        else unit.group(0),
                precisedelta(
                    timedelta(seconds=v),
                    minimum_unit="microseconds",
                )
            ).replace(" and", "")
        ))

    g.figure.suptitle(name, va="bottom")

    if tight_layout:
        g.figure.tight_layout()

    return g

End-to-end

g = plot_benchmark(
    "test_benchmark_infer_end_to_end_real",
    ["benchmark_real_precursors", "heuristic"],
    bench_df = _bench_df
        .with_columns(
            pl.col("name")
                .str.extract_groups(r"^(?<name>.*?)(?:\[(?<size>\d+)-(?<heuristic>.+?)\])?$")
                .alias("name_parsed"),
        )
        .drop("name")
        .unnest("name_parsed", "params", "stats"),
    method="rel",
    kind="line",
    estimator=None, n_boot=0,
    tight_layout=False,
    legend=True,
    facet_kws=dict(
        sharey=True,
    ),
    alpha=0.67,
)

Core inference functions

g = plot_benchmark(
    "test_benchmark_infer_sparse_graph_mock",
    ["benchmark_mock_edges", "inference_method", "infer_form", "sparray_fmt"],
    method="rel",
    kind="line",
    estimator=None, n_boot=0,
    tight_layout=False,
    legend=True,
    facet_kws=dict(
        sharey=True,
    ),
)

g = plot_benchmark(
    "test_benchmark_infer_sparse_graph_mock",
    ["benchmark_mock_edges", "inference_method", "sparray_fmt", "infer_form"],
    method="rel",
    kind="line",
    estimator=None, n_boot=0,
    tight_layout=False,
    legend=True,
    facet_kws=dict(
        sharey=True,
    ),
)

g = plot_benchmark(
    "test_benchmark_infer_sparse_graph_real",
    ["benchmark_real_precursors", "inference_method", "infer_form", "sparray_fmt"],
    method="rel",
    kind="line",
    estimator=None, n_boot=0,
    tight_layout=False,
    legend=True,
    facet_kws=dict(
        sharey=True,
    ),
)

g = plot_benchmark(
    "test_benchmark_infer_sparse_graph_real",
    ["benchmark_real_precursors", "inference_method", "sparray_fmt", "infer_form"],
    method="rel",
    kind="line",
    estimator=None, n_boot=0,
    tight_layout=False,
    legend=True,
    facet_kws=dict(
        sharey=True,
    ),
)

g = plot_benchmark(
    "test_benchmark_compute_proteotypicity_mock",
    ["benchmark_mock_edges"],
    method="rel",
    kind="line",
    estimator=None, n_boot=0,
    tight_layout=False,
    legend=True,
    facet_kws=dict(
        sharey=True,
    ),
)

g = plot_benchmark(
    "test_benchmark_compute_proteotypicity_real",
    ["benchmark_real_precursors"],
    method="rel",
    kind="line",
    estimator=None, n_boot=0,
    tight_layout=False,
    legend=True,
    facet_kws=dict(
        sharey=True,
    ),
)

Graph manipulation

plot_benchmark(
    "test_benchmark_edges_to_peptides",
    "benchmark_mock_edges",
    method="rel",
    kind="line",
    n_boot=None,
);

plot_benchmark(
    "test_benchmark_build_sparse_graph_mock",
    "benchmark_mock_edges",
    method="rel",
    kind="line",
    n_boot=None,
);

plot_benchmark(
    "test_benchmark_build_sparse_graph_real",
    "benchmark_real_precursors",
    method="rel",
    kind="line",
    n_boot=None,
);

plot_benchmark(
    "test_benchmark_get_connected_components_mock",
    ["benchmark_mock_edges", "sparray_fmt"],
    method="rel",
    kind="line",
    n_boot=None,
    tight_layout=False,
);

plot_benchmark(
    "test_benchmark_get_connected_components_real",
    ["benchmark_real_precursors", "sparray_fmt"],
    method="rel",
    kind="line",
    n_boot=None,
    tight_layout=False,
);