Faster Multi-Object Segmentation using Parallel Quadratic Pseudo-Boolean Optimization, ICCV 2021 Paper
Author: Niels Jeppesen (niejep@dtu.dk)
This notebook is based on the NerveSegmentation3D.ipynb notebook by Jeppesen et. al. Parts of this notebook is copied directly from the NerveSegmentation3D.ipynb notebook with little or no modification.
The goal of this notebook is to benchmark three different QPBO implementations (P-QPBO, M-QPBO and K-QPBO) on two large 3D segmentation tasks.
To run the benchmark, a modified version of the slgbuilder package, which includes P-QPBO and M-QPBO is required. This version of the slgbuilder package relies on the shrdr package for P-QPBO and M-QPBO C++ wrappers. Both packages are included in the supplementary material alongside this notebook.
from datetime import datetime
import os
import platform
import time
import nibabel as nib
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from scipy.ndimage import filters, interpolation
from scipy.spatial.transform import Rotation as R
from sklearn import neighbors
from tqdm import tqdm
from slgbuilder import GraphObject, QPBOBuilder, MQPBOBuilder, PQPBOBuilder
The N1 and N2 segmentation tasks are based on the same raw data. However, N2 uses more samples, resulting in a larger graph. The two cells below contain the paramters used for the N1 and N2 tasks, respectively. For N2 we have use to 64-bit edge/arc indices to avoid overflow due to the number of edges.
# N1
# arc_index_type = np.uint32
# runs_per_config = 10
# steps = 40
# angles = 30
# N2
arc_index_type = np.uint64
runs_per_config = 3
steps = 60
angles = 45
The data we'll be working with comes from volume of a small cut-out from the hand or wrist of a person. In this data, the nerves are clearly visible and almost completely aligned in terms of direction. In this notebook we'll working with the full volume and segment our over one hundred nerves. For the nerves we'll be segmenting center lines have been annotated by a another process.
The raw volume is 2048x2048x2048. The center annotations were made on a 512x512x512 downscaled volume, but we can use them here.
Let's load the image and annotated approximate center positions.
data_dir = '../originals'
results_dir = '../benchmark'
data_path = os.path.join(data_dir, 'NT32_tomo3_.raw')
centers_path = os.path.join(data_dir, 'NT32_cLineLabel_scale4_preSegm_v3.nii.gz')
data_centers_scale = 4
data = np.memmap(data_path, shape=(2048, 2048, 2048), dtype=np.uint8, mode='c')
ni_centers = nib.load(centers_path)
centers_data = np.asanyarray(ni_centers.dataobj).transpose()
# Show two slices.
plt.figure(figsize=(20, 20))
ax = plt.subplot(1, 2, 1, title='Slice 800')
ax.imshow(data[800].T, cmap='gray', vmin=0, vmax=255)
ax = plt.subplot(1, 2, 2, title='Slice 1600')
ax.imshow(data[1600].T, cmap='gray', vmin=0, vmax=255)
plt.show()
Because the center positions where created on a lower resolution we need to translate their positions and use interpolation to estimates centers in the slices we create. Each nerve will have a center line, which we will create as a simple list of points.
nerve_labels = np.unique(centers_data)[1:]
center_lines_original = {l: [] for l in nerve_labels}
center_lines = {l: [] for l in nerve_labels}
# For each slice, create points for each labelled pixel.
for x, center_slice in enumerate(tqdm(centers_data)):
ys, zs = np.where(center_slice)
for y, z in zip(ys, zs):
center_lines_original[center_slice[y, z]].append((x, y, z))
# Center positions may be more than one pixel per slice.
# To ensure we have only a single point per label per slice, we group by x (slice) position and take the mean value.
for label in center_lines_original:
label_points = pd.DataFrame(center_lines_original[label]).groupby(0).mean() * data_centers_scale
center_lines[label] = np.concatenate([label_points.index.values.astype(np.float32)[..., np.newaxis] * data_centers_scale, label_points.to_numpy().astype(np.float32)], axis=-1)
100%|██████████| 512/512 [00:01<00:00, 320.02it/s]
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1, projection='3d')
for label in center_lines:
line = center_lines[label]
ax.plot(line[:, 2], line[:, 1], line[:, 0])
ax.view_init(30, 45)
plt.show()
plt.figure(figsize=(15, 15))
ax = plt.subplot(1, 1, 1)
ax.imshow(data[1000].T, cmap='gray')
for label in center_lines:
line = center_lines[label]
for center in line:
if center[0] == 1000:
ax.scatter(center[1], center[2])
plt.show()
We need unfold the nerves to use them with column-ordered graphs.
def angle(v1, v2):
cosang = np.dot(v1, v2)
sinang = np.linalg.norm(np.cross(v1, v2))
return np.arctan2(sinang, cosang)
def unfold_around_centers(data, centers, r_min=2, r_max=50, angles=40, steps=30, prefilter=False):
"""Radially samples an image around a given center position with a given radius and resolution."""
# Sampling angles and radii.
angles = np.linspace(0, 2*np.pi, angles, endpoint=False, dtype=np.float32)
distances = np.linspace(r_min, r_max, steps, endpoint=True, dtype=np.float32)
# Get angles.
angles_cos = np.cos(angles)
angles_sin = np.sin(angles)
# Calculate points positions.
y_pos = np.outer(angles_cos, distances)
z_pos = np.outer(angles_sin, distances)
# Smooth center line before taking gradient to avoid sharp bends.
grad = np.gradient(filters.gaussian_filter(centers, sigma=(5, 0)), axis=0)
samples_vol = []
sampling_points_vol = []
for norm, center in zip(grad, centers):
sampling_points = np.array([np.zeros(y_pos.shape, dtype=np.float32), y_pos, z_pos]).transpose()
# Create list of sampling points.
sampling_shape = sampling_points.shape
sampling_points_flat = sampling_points.reshape((-1, 3))
v_x = np.array([1.0, 0, 0])
norm /= np.linalg.norm(norm)
diff = np.cross(v_x, norm)
diff_norm = np.linalg.norm(diff)
if diff_norm > 0:
diff /= diff_norm
diff *= angle(v_x, norm)
rot = R.from_rotvec(diff)
sampling_points_flat = rot.apply(sampling_points_flat)
# Translate to real position.
sampling_points_flat += center[np.newaxis]
sampling_points = sampling_points_flat.reshape(sampling_shape)
# Sample from image.
samples = interpolation.map_coordinates(data, sampling_points_flat.transpose(), mode='nearest', prefilter=prefilter)
samples = samples.reshape(sampling_shape[:2])
samples_vol.append(samples)
sampling_points_vol.append(sampling_points)
return np.asarray(samples_vol), np.asarray(sampling_points_vol)
s, sp = unfold_around_centers(data, center_lines[1], r_min=5, r_max=80, steps=25, angles=30)
fig = plt.figure(figsize=(15, 15))
plt.scatter(sp[256, ..., 2], sp[256, ..., 1], c=s[256], s=100, cmap='gray')
plt.show()
plt.figure(figsize=(20, 7))
for i in range(24):
ax = plt.subplot(3, 8, i + 1)
ax.imshow(s[i * 20], cmap='gray')
plt.show()
We define functions for constructing the energy function using slgbuilder, specifically the MQPBOBuilder. We also define functions for copying a builder, building and solving the defined QPBO problem. The reason we copy the builder is to save time in the build process.
def build_and_solve(helper):
build_time = None
try:
start = time.time()
helper.build_graph()
build_time = time.time() - start
except Exception as ex:
print(str(ex))
raise ex
solve_time = None
weak_persistencies_time = None
if build_time is not None:
try:
start = time.time()
helper.solve(compute_weak_persistencies=False)
solve_time = time.time() - start
start = time.time()
helper.graph.compute_weak_persistencies()
weak_persistencies_time = time.time() - start
twice_energy = helper.graph.compute_twice_energy()
except Exception as ex:
print(str(ex))
raise ex
return twice_energy, build_time, solve_time, weak_persistencies_time
def print_results(stats):
s = '%s:\nNodes: %s, Terms: %s, Build: %s, Solve: %s, Weak persistencies: %s, Twice energy: %s, Timstamp: %s'
print(s % tuple(stats))
def create_qpbo_builder(data, center_lines, layered_smoothness=2, containment_min_margin=10, containment_max_margin=35, exclusion_margin=3, capacity_type=np.int32, arc_index_type=np.uint32):
# Lists for storing nerve objects.
outer_nerves = []
inner_nerves = []
# For each center, create an inner and outer never.
for centers in tqdm(center_lines, desc='Creating objects'):
# Unfold nerve.
samples, sample_points = unfold_around_centers(data, centers, r_min=5, r_max=80, steps=steps, angles=angles)
# Move x-axis last.
samples = np.moveaxis(samples, 0, -1).astype(np.float32)
sample_points = np.moveaxis(sample_points, 0, -2)
# Create outer and inner nerve objects.
outer_diff_samples = np.gradient(-samples, axis=0)
outer_diff_sample_points = sample_points
inner_diff_samples = np.gradient(samples, axis=0)
inner_diff_sample_points = sample_points
outer_nerves.append(GraphObject(outer_diff_samples, outer_diff_sample_points))
inner_nerves.append(GraphObject(inner_diff_samples, inner_diff_sample_points))
estimated_nodes = sum(o.data.size for o in outer_nerves) + sum(o.data.size for o in inner_nerves)
helper = MQPBOBuilder(estimated_nodes, 0, jit_build=True, capacity_type=capacity_type, arc_index_type=arc_index_type)
for outer_nerve, inner_nerve in zip(outer_nerves, inner_nerves):
helper.add_objects([outer_nerve, inner_nerve])
helper.add_layered_boundary_cost()
wrap = np.ones((len(helper.objects), 2), dtype=np.bool)
wrap[:, -1] = False
helper.add_layered_smoothness(delta=layered_smoothness, wrap=wrap)
for outer_nerve, inner_nerve in tqdm(zip(outer_nerves, inner_nerves), total=len(outer_nerves), desc='Containment'):
helper.add_layered_containment(outer_nerve, inner_nerve, min_margin=containment_min_margin, max_margin=containment_max_margin, distance_metric='l2')
# Add exclusion constraints between all pairs of outer nerves.
if exclusion_margin is not None:
start = time.time()
exclusions_dic = {}
for i in range(len(outer_nerves)):
ex_list = []
neigh = neighbors.NearestNeighbors(radius=80*2 + exclusion_margin, metric='l2')
neigh.fit(center_lines[i])
for j in range(i + 1, len(outer_nerves)):
neigh_indices = neigh.radius_neighbors(center_lines[j], return_distance=False)
if neigh_indices[0].size > 0:
ex_list.append(outer_nerves[j])
if ex_list:
exclusions_dic[outer_nerves[i]] = ex_list
helper.add_layered_exclusions(exclusions_dic, margin=exclusion_margin, distance_metric='l2')
print(f'Exclusion: {round(time.time() - start, 3)}')
node_count = 2 * np.sum([o.data.size for o in helper.objects])
edge_count = 2 * np.sum([l.size for l in helper.pairwise_from])
print(f'Graph has {node_count} nodes and {edge_count} terms.')
return helper
def copy_and_solve(builder, builder_fn):
# Create new builder.
helper = builder_fn(0, 0)
# Copy graph structure from previous builder.
helper.objects = builder.objects[:]
helper.nodes = builder.nodes[:]
helper.unary_nodes = builder.unary_nodes[:]
helper.unary_e0 = builder.unary_e0[:]
helper.unary_e1 = builder.unary_e1[:]
helper.pairwise_from = builder.pairwise_from[:]
helper.pairwise_to = builder.pairwise_to[:]
helper.pairwise_e00 = builder.pairwise_e00[:]
helper.pairwise_e01 = builder.pairwise_e01[:]
helper.pairwise_e10 = builder.pairwise_e10[:]
helper.pairwise_e11 = builder.pairwise_e11[:]
node_count = 2 * np.sum([o.data.size for o in helper.objects])
edge_count = 2 * np.sum([l.size for l in helper.pairwise_from])
# Build and solve.
twice_energy, build_time, solve_time, weak_persistencies_time = build_and_solve(helper)
return helper, [type(helper.graph).__name__, node_count, edge_count, build_time, solve_time, weak_persistencies_time, twice_energy, datetime.now()]
We solve QPBO problem with each of the three different implementations a number of times. For P-QPBO we benchmark the implementation for a number of different parallel thread configurations.
capacity_type = np.int32
cl = [center_lines[l] for l in list(center_lines.keys())]
reference_helper = create_qpbo_builder(data, cl, capacity_type=capacity_type, arc_index_type=arc_index_type)
Creating objects: 100%|██████████| 216/216 [02:58<00:00, 1.21it/s] Containment: 100%|██████████| 216/216 [00:05<00:00, 39.07it/s]
Exclusion: 288.831 Graph has 818434800 nodes and 4864255488 terms.
cpu_counts = [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64]
parallel_builders = [PQPBOBuilder]
serial_builders = [QPBOBuilder, MQPBOBuilder]
helper_stats_list = []
cpu_counts_list = []
for builder in parallel_builders:
for cpu_count in cpu_counts:
for i in range(runs_per_config):
print(f'\n{builder.__name__} - cpus: {cpu_count}, run: {i + 1}/{runs_per_config}')
# Create builder function.
builder_fn = lambda estimated_nodes, estimated_terms:builder(estimated_nodes, estimated_terms, capacity_type=capacity_type, arc_index_type=arc_index_type, num_threads=cpu_count)
# Build and solve.
helper, helper_stats = copy_and_solve(reference_helper, builder_fn)
# Print segmentation time and graph stats.
print_results(helper_stats)
# Clear memory.
del helper
# Save stats.
helper_stats_list.append(helper_stats)
cpu_counts_list.append(cpu_count)
for builder in serial_builders:
for i in range(runs_per_config):
print(f'\n{builder.__name__} - run: {i + 1}/{runs_per_config}')
# Create builder function.
if builder == QPBOBuilder:
builder_fn = lambda estimated_nodes, estimated_terms:builder(estimated_nodes, estimated_terms, capacity_type=capacity_type)
else:
builder_fn = lambda estimated_nodes, estimated_terms:builder(estimated_nodes, estimated_terms, capacity_type=capacity_type, arc_index_type=arc_index_type)
# Build and solve.
helper, helper_stats = copy_and_solve(reference_helper, builder_fn)
# Print segmentation time and graph stats.
print_results(helper_stats)
# Clear memory.
del helper
# Save stats.
helper_stats_list.append(helper_stats)
cpu_counts_list.append(-1)
PQPBOBuilder - cpus: 1, run: 1/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 201.74432373046875, Solve: 2429.0122425556183, Weak persistencies: 23.836749792099, Twice energy: 311358216, Timstamp: 2021-06-17 01:42:16.374891 PQPBOBuilder - cpus: 1, run: 2/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 212.9748876094818, Solve: 2545.285952091217, Weak persistencies: 23.950043201446533, Twice energy: 311358216, Timstamp: 2021-06-17 02:29:51.969108 PQPBOBuilder - cpus: 1, run: 3/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 207.06010007858276, Solve: 2620.5860874652863, Weak persistencies: 30.305575609207153, Twice energy: 311358216, Timstamp: 2021-06-17 03:18:51.883301 PQPBOBuilder - cpus: 2, run: 1/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 208.7583200931549, Solve: 1370.1748750209808, Weak persistencies: 22.58077836036682, Twice energy: 311358216, Timstamp: 2021-06-17 03:46:40.130511 PQPBOBuilder - cpus: 2, run: 2/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 205.32350039482117, Solve: 1376.9441437721252, Weak persistencies: 21.912545442581177, Twice energy: 311358216, Timstamp: 2021-06-17 04:14:26.739481 PQPBOBuilder - cpus: 2, run: 3/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 208.48932003974915, Solve: 1376.2618975639343, Weak persistencies: 21.42883062362671, Twice energy: 311358216, Timstamp: 2021-06-17 04:42:13.936504 PQPBOBuilder - cpus: 4, run: 1/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 205.62064814567566, Solve: 797.2288112640381, Weak persistencies: 21.384308338165283, Twice energy: 311358216, Timstamp: 2021-06-17 05:00:11.368620 PQPBOBuilder - cpus: 4, run: 2/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 205.22260117530823, Solve: 794.9546213150024, Weak persistencies: 21.564449310302734, Twice energy: 311358216, Timstamp: 2021-06-17 05:18:06.752827 PQPBOBuilder - cpus: 4, run: 3/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 205.87564301490784, Solve: 796.9030611515045, Weak persistencies: 22.108452796936035, Twice energy: 311358216, Timstamp: 2021-06-17 05:36:11.166190 PQPBOBuilder - cpus: 8, run: 1/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 205.3399384021759, Solve: 471.1951720714569, Weak persistencies: 21.76146173477173, Twice energy: 311358216, Timstamp: 2021-06-17 05:48:48.502887 PQPBOBuilder - cpus: 8, run: 2/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 208.0752191543579, Solve: 471.01614928245544, Weak persistencies: 22.176071405410767, Twice energy: 311358216, Timstamp: 2021-06-17 06:01:27.953093 PQPBOBuilder - cpus: 8, run: 3/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 205.61123061180115, Solve: 474.72411370277405, Weak persistencies: 21.209575653076172, Twice energy: 311358216, Timstamp: 2021-06-17 06:14:09.605700 PQPBOBuilder - cpus: 16, run: 1/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 207.34222316741943, Solve: 326.6416642665863, Weak persistencies: 24.353436946868896, Twice energy: 311358216, Timstamp: 2021-06-17 06:24:30.150836 PQPBOBuilder - cpus: 16, run: 2/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 205.8108651638031, Solve: 325.9785752296448, Weak persistencies: 21.801130533218384, Twice energy: 311358216, Timstamp: 2021-06-17 06:34:48.281791 PQPBOBuilder - cpus: 16, run: 3/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 205.87650299072266, Solve: 322.8505115509033, Weak persistencies: 23.715320825576782, Twice energy: 311358216, Timstamp: 2021-06-17 06:45:02.224740 PQPBOBuilder - cpus: 24, run: 1/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 207.27886819839478, Solve: 295.8663845062256, Weak persistencies: 26.52097225189209, Twice energy: 311358216, Timstamp: 2021-06-17 06:55:01.520454 PQPBOBuilder - cpus: 24, run: 2/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 211.25870490074158, Solve: 276.6243705749512, Weak persistencies: 22.192971229553223, Twice energy: 311358216, Timstamp: 2021-06-17 07:04:32.017141 PQPBOBuilder - cpus: 24, run: 3/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 207.7742052078247, Solve: 282.40342473983765, Weak persistencies: 21.930246829986572, Twice energy: 311358216, Timstamp: 2021-06-17 07:13:58.020524 PQPBOBuilder - cpus: 32, run: 1/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 207.08836936950684, Solve: 263.8761281967163, Weak persistencies: 23.394057750701904, Twice energy: 311358216, Timstamp: 2021-06-17 07:23:26.369140 PQPBOBuilder - cpus: 32, run: 2/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 207.25813484191895, Solve: 270.9674413204193, Weak persistencies: 23.311651706695557, Twice energy: 311358216, Timstamp: 2021-06-17 07:32:48.131756 PQPBOBuilder - cpus: 32, run: 3/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 207.52311897277832, Solve: 269.3863899707794, Weak persistencies: 23.719178676605225, Twice energy: 311358216, Timstamp: 2021-06-17 07:42:07.931167 PQPBOBuilder - cpus: 40, run: 1/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 206.5893054008484, Solve: 249.93723464012146, Weak persistencies: 23.561327934265137, Twice energy: 311358216, Timstamp: 2021-06-17 07:51:06.404451 PQPBOBuilder - cpus: 40, run: 2/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 205.52096509933472, Solve: 250.43844866752625, Weak persistencies: 23.63571333885193, Twice energy: 311358216, Timstamp: 2021-06-17 08:00:04.653388 PQPBOBuilder - cpus: 40, run: 3/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 206.14481687545776, Solve: 244.62009453773499, Weak persistencies: 23.502421379089355, Twice energy: 311358216, Timstamp: 2021-06-17 08:08:56.776878 PQPBOBuilder - cpus: 48, run: 1/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 207.2906219959259, Solve: 247.1417899131775, Weak persistencies: 23.543741703033447, Twice energy: 311358216, Timstamp: 2021-06-17 08:17:51.908152 PQPBOBuilder - cpus: 48, run: 2/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 203.27904343605042, Solve: 239.07189226150513, Weak persistencies: 26.152801990509033, Twice energy: 311358216, Timstamp: 2021-06-17 08:26:49.545814 PQPBOBuilder - cpus: 48, run: 3/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 239.77185773849487, Solve: 254.36127495765686, Weak persistencies: 21.23779797554016, Twice energy: 311358216, Timstamp: 2021-06-17 08:36:23.454662 PQPBOBuilder - cpus: 56, run: 1/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 207.9782531261444, Solve: 257.9021441936493, Weak persistencies: 25.576632499694824, Twice energy: 311358216, Timstamp: 2021-06-17 08:45:40.875430 PQPBOBuilder - cpus: 56, run: 2/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 196.43799805641174, Solve: 243.65397596359253, Weak persistencies: 22.475586652755737, Twice energy: 311358216, Timstamp: 2021-06-17 08:54:33.403182 PQPBOBuilder - cpus: 56, run: 3/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 231.83338046073914, Solve: 260.39250469207764, Weak persistencies: 21.486783981323242, Twice energy: 311358216, Timstamp: 2021-06-17 09:04:01.597680 PQPBOBuilder - cpus: 64, run: 1/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 210.45083165168762, Solve: 272.0539093017578, Weak persistencies: 26.714988470077515, Twice energy: 311358216, Timstamp: 2021-06-17 09:13:32.156695 PQPBOBuilder - cpus: 64, run: 2/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 201.58857226371765, Solve: 240.4371213912964, Weak persistencies: 21.516854524612427, Twice energy: 311358216, Timstamp: 2021-06-17 09:22:16.325956 PQPBOBuilder - cpus: 64, run: 3/3 ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 242.1808681488037, Solve: 256.90800285339355, Weak persistencies: 22.015126943588257, Twice energy: 311358216, Timstamp: 2021-06-17 09:32:00.250681 QPBOBuilder - run: 1/3 QPBOInt: Nodes: 818434800, Terms: 4864255488, Build: 765.7046883106232, Solve: 5056.997569561005, Weak persistencies: 32.05951261520386, Twice energy: 311358216, Timstamp: 2021-06-17 11:11:17.210416 QPBOBuilder - run: 2/3 QPBOInt: Nodes: 818434800, Terms: 4864255488, Build: 897.128082036972, Solve: 5340.008810520172, Weak persistencies: 37.70053768157959, Twice energy: 311358216, Timstamp: 2021-06-17 12:57:57.881656 QPBOBuilder - run: 3/3 QPBOInt: Nodes: 818434800, Terms: 4864255488, Build: 862.8035168647766, Solve: 4986.645044803619, Weak persistencies: 25.902894258499146, Twice energy: 311358216, Timstamp: 2021-06-17 14:37:41.872564 MQPBOBuilder - run: 1/3 QpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 107.16999554634094, Solve: 3746.184516429901, Weak persistencies: 28.300907135009766, Twice energy: 14876534453474, Timstamp: 2021-06-17 15:43:38.488163 MQPBOBuilder - run: 2/3 QpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 130.1824414730072, Solve: 3868.691021680832, Weak persistencies: 15.89573359489441, Twice energy: 14876534453474, Timstamp: 2021-06-17 16:51:21.945680 MQPBOBuilder - run: 3/3 QpboCapInt32ArcIdxUInt64NodeIdxUInt32: Nodes: 818434800, Terms: 4864255488, Build: 114.69807505607605, Solve: 3703.7911460399628, Weak persistencies: 14.528364896774292, Twice energy: 14876534453474, Timstamp: 2021-06-17 17:56:12.716527
We add system information to the results and save them as CSV.
df = pd.DataFrame(data=helper_stats_list, columns=['Class', 'NodeCount', 'EdgeCount', 'BuildTime', 'SolveTime', 'WeakPersistenciesTime', 'TwiceEnergy', 'Timestamp'])
df['CpuCount'] = np.array(cpu_counts_list, dtype=np.int16)
df['ShortName'] = df['Class'].str[:4]
df.loc[df['CpuCount'] != -1, 'ShortName'] += ' (' + df['CpuCount'].astype(np.str) + ')'
df['SystemName'] = platform.node()
try:
import cpuinfo
info = cpuinfo.get_cpu_info()
df['SystemCpu'] = info['brand_raw']
except:
df['SystemCpu'] = platform.processor()
df['SystemCpuCount'] = np.array(os.cpu_count(), np.int16)
df['NerveCount'] = np.array(len(cl), np.int16)
df_full = df
df_full
| Class | NodeCount | EdgeCount | BuildTime | SolveTime | WeakPersistenciesTime | TwiceEnergy | Timestamp | CpuCount | ShortName | SystemName | SystemCpu | SystemCpuCount | NerveCount | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 201.744324 | 2429.012243 | 23.836750 | 311358216 | 2021-06-17 01:42:16.374891 | 1 | Para (1) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 1 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 212.974888 | 2545.285952 | 23.950043 | 311358216 | 2021-06-17 02:29:51.969108 | 1 | Para (1) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 2 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 207.060100 | 2620.586087 | 30.305576 | 311358216 | 2021-06-17 03:18:51.883301 | 1 | Para (1) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 3 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 208.758320 | 1370.174875 | 22.580778 | 311358216 | 2021-06-17 03:46:40.130511 | 2 | Para (2) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 4 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 205.323500 | 1376.944144 | 21.912545 | 311358216 | 2021-06-17 04:14:26.739481 | 2 | Para (2) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 5 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 208.489320 | 1376.261898 | 21.428831 | 311358216 | 2021-06-17 04:42:13.936504 | 2 | Para (2) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 6 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 205.620648 | 797.228811 | 21.384308 | 311358216 | 2021-06-17 05:00:11.368620 | 4 | Para (4) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 7 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 205.222601 | 794.954621 | 21.564449 | 311358216 | 2021-06-17 05:18:06.752827 | 4 | Para (4) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 8 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 205.875643 | 796.903061 | 22.108453 | 311358216 | 2021-06-17 05:36:11.166190 | 4 | Para (4) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 9 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 205.339938 | 471.195172 | 21.761462 | 311358216 | 2021-06-17 05:48:48.502887 | 8 | Para (8) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 10 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 208.075219 | 471.016149 | 22.176071 | 311358216 | 2021-06-17 06:01:27.953093 | 8 | Para (8) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 11 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 205.611231 | 474.724114 | 21.209576 | 311358216 | 2021-06-17 06:14:09.605700 | 8 | Para (8) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 12 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 207.342223 | 326.641664 | 24.353437 | 311358216 | 2021-06-17 06:24:30.150836 | 16 | Para (16) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 13 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 205.810865 | 325.978575 | 21.801131 | 311358216 | 2021-06-17 06:34:48.281791 | 16 | Para (16) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 14 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 205.876503 | 322.850512 | 23.715321 | 311358216 | 2021-06-17 06:45:02.224740 | 16 | Para (16) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 15 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 207.278868 | 295.866385 | 26.520972 | 311358216 | 2021-06-17 06:55:01.520454 | 24 | Para (24) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 16 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 211.258705 | 276.624371 | 22.192971 | 311358216 | 2021-06-17 07:04:32.017141 | 24 | Para (24) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 17 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 207.774205 | 282.403425 | 21.930247 | 311358216 | 2021-06-17 07:13:58.020524 | 24 | Para (24) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 18 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 207.088369 | 263.876128 | 23.394058 | 311358216 | 2021-06-17 07:23:26.369140 | 32 | Para (32) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 19 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 207.258135 | 270.967441 | 23.311652 | 311358216 | 2021-06-17 07:32:48.131756 | 32 | Para (32) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 20 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 207.523119 | 269.386390 | 23.719179 | 311358216 | 2021-06-17 07:42:07.931167 | 32 | Para (32) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 21 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 206.589305 | 249.937235 | 23.561328 | 311358216 | 2021-06-17 07:51:06.404451 | 40 | Para (40) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 22 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 205.520965 | 250.438449 | 23.635713 | 311358216 | 2021-06-17 08:00:04.653388 | 40 | Para (40) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 23 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 206.144817 | 244.620095 | 23.502421 | 311358216 | 2021-06-17 08:08:56.776878 | 40 | Para (40) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 24 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 207.290622 | 247.141790 | 23.543742 | 311358216 | 2021-06-17 08:17:51.908152 | 48 | Para (48) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 25 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 203.279043 | 239.071892 | 26.152802 | 311358216 | 2021-06-17 08:26:49.545814 | 48 | Para (48) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 26 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 239.771858 | 254.361275 | 21.237798 | 311358216 | 2021-06-17 08:36:23.454662 | 48 | Para (48) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 27 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 207.978253 | 257.902144 | 25.576632 | 311358216 | 2021-06-17 08:45:40.875430 | 56 | Para (56) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 28 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 196.437998 | 243.653976 | 22.475587 | 311358216 | 2021-06-17 08:54:33.403182 | 56 | Para (56) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 29 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 231.833380 | 260.392505 | 21.486784 | 311358216 | 2021-06-17 09:04:01.597680 | 56 | Para (56) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 30 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 210.450832 | 272.053909 | 26.714988 | 311358216 | 2021-06-17 09:13:32.156695 | 64 | Para (64) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 31 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 201.588572 | 240.437121 | 21.516855 | 311358216 | 2021-06-17 09:22:16.325956 | 64 | Para (64) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 32 | ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 242.180868 | 256.908003 | 22.015127 | 311358216 | 2021-06-17 09:32:00.250681 | 64 | Para (64) | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 33 | QPBOInt | 818434800 | 4864255488 | 765.704688 | 5056.997570 | 32.059513 | 311358216 | 2021-06-17 11:11:17.210416 | -1 | QPBO | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 34 | QPBOInt | 818434800 | 4864255488 | 897.128082 | 5340.008811 | 37.700538 | 311358216 | 2021-06-17 12:57:57.881656 | -1 | QPBO | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 35 | QPBOInt | 818434800 | 4864255488 | 862.803517 | 4986.645045 | 25.902894 | 311358216 | 2021-06-17 14:37:41.872564 | -1 | QPBO | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 36 | QpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 107.169996 | 3746.184516 | 28.300907 | 14876534453474 | 2021-06-17 15:43:38.488163 | -1 | Qpbo | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 37 | QpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 130.182441 | 3868.691022 | 15.895734 | 14876534453474 | 2021-06-17 16:51:21.945680 | -1 | Qpbo | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
| 38 | QpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 818434800 | 4864255488 | 114.698075 | 3703.791146 | 14.528365 | 14876534453474 | 2021-06-17 17:56:12.716527 | -1 | Qpbo | n-62-11-52 | Intel(R) Xeon(R) Gold 6226R CPU @ 2.90GHz | 32 | 216 |
timestr = time.strftime("%Y%m%d-%H%M%S")
df_full.to_csv(os.path.join(results_dir, f'parallel_qpbo_benchmark_results_{timestr}.csv'))
df_group = df.groupby(['Class', 'CpuCount'])
df_group[['BuildTime', 'SolveTime']].describe()
| BuildTime | SolveTime | ||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | mean | std | min | 25% | 50% | 75% | max | count | mean | std | min | 25% | 50% | 75% | max | ||
| Class | CpuCount | ||||||||||||||||
| ParallelQpboCapInt32ArcIdxUInt64NodeIdxUInt32 | 1 | 3.0 | 207.259770 | 5.617944 | 201.744324 | 204.402212 | 207.060100 | 210.017494 | 212.974888 | 3.0 | 2531.628094 | 96.514441 | 2429.012243 | 2487.149097 | 2545.285952 | 2582.936020 | 2620.586087 |
| 2 | 3.0 | 207.523714 | 1.910182 | 205.323500 | 206.906410 | 208.489320 | 208.623820 | 208.758320 | 3.0 | 1374.460305 | 3.726936 | 1370.174875 | 1373.218386 | 1376.261898 | 1376.603021 | 1376.944144 | |
| 4 | 3.0 | 205.572964 | 0.329122 | 205.222601 | 205.421625 | 205.620648 | 205.748146 | 205.875643 | 3.0 | 796.362165 | 1.229802 | 794.954621 | 795.928841 | 796.903061 | 797.065936 | 797.228811 | |
| 8 | 3.0 | 206.342129 | 1.507017 | 205.339938 | 205.475585 | 205.611231 | 206.843225 | 208.075219 | 3.0 | 472.311812 | 2.091032 | 471.016149 | 471.105661 | 471.195172 | 472.959643 | 474.724114 | |
| 16 | 3.0 | 206.343197 | 0.865804 | 205.810865 | 205.843684 | 205.876503 | 206.609363 | 207.342223 | 3.0 | 325.156917 | 2.024735 | 322.850512 | 324.414543 | 325.978575 | 326.310120 | 326.641664 | |
| 24 | 3.0 | 208.770593 | 2.168955 | 207.278868 | 207.526537 | 207.774205 | 209.516455 | 211.258705 | 3.0 | 284.964727 | 9.873397 | 276.624371 | 279.513898 | 282.403425 | 289.134905 | 295.866385 | |
| 32 | 3.0 | 207.289874 | 0.219106 | 207.088369 | 207.173252 | 207.258135 | 207.390627 | 207.523119 | 3.0 | 268.076653 | 3.722665 | 263.876128 | 266.631259 | 269.386390 | 270.176916 | 270.967441 | |
| 40 | 3.0 | 206.085029 | 0.536674 | 205.520965 | 205.832891 | 206.144817 | 206.367061 | 206.589305 | 3.0 | 248.331926 | 3.224294 | 244.620095 | 247.278665 | 249.937235 | 250.187842 | 250.438449 | |
| 48 | 3.0 | 216.780508 | 20.011867 | 203.279043 | 205.284833 | 207.290622 | 223.531240 | 239.771858 | 3.0 | 246.858319 | 7.648632 | 239.071892 | 243.106841 | 247.141790 | 250.751532 | 254.361275 | |
| 56 | 3.0 | 212.083211 | 18.051213 | 196.437998 | 202.208126 | 207.978253 | 219.905817 | 231.833380 | 3.0 | 253.982875 | 9.031339 | 243.653976 | 250.778060 | 257.902144 | 259.147324 | 260.392505 | |
| 64 | 3.0 | 218.073424 | 21.342717 | 201.588572 | 206.019702 | 210.450832 | 226.315850 | 242.180868 | 3.0 | 256.466345 | 15.813020 | 240.437121 | 248.672562 | 256.908003 | 264.480956 | 272.053909 | |
| QPBOInt | -1 | 3.0 | 841.878762 | 68.164588 | 765.704688 | 814.254103 | 862.803517 | 879.965799 | 897.128082 | 3.0 | 5127.883808 | 187.043128 | 4986.645045 | 5021.821307 | 5056.997570 | 5198.503190 | 5340.008811 |
| QpboCapInt32ArcIdxUInt64NodeIdxUInt32 | -1 | 3.0 | 117.350171 | 11.733217 | 107.169996 | 110.934035 | 114.698075 | 122.440258 | 130.182441 | 3.0 | 3772.888895 | 85.631975 | 3703.791146 | 3724.987831 | 3746.184516 | 3807.437769 | 3868.691022 |