In [2]:
# Set up sys.path so that 'src/spindle_dev' is importable as 'spindle_dev'
import sys
import importlib
from pathlib import Path
project_root = '/data/sarkar_lab/Projects/spindle_dev'
src_path = Path(project_root) / 'src'
if str(src_path) not in sys.path:
sys.path.insert(0, str(src_path))
import spindle_dev
# Reload to pick up code changes without restarting the kernel
importlib.reload(spindle_dev)
Out[2]:
<module 'spindle_dev' from '/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/__init__.py'>
In [3]:
import scanpy as sc
In [4]:
import glob
In [5]:
h5ad_files = glob.glob("/data/sarkar_lab/insitupy_demo_data_xenium/*.h5ad")
In [6]:
h5ad_files
Out[6]:
['/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_skin_melanoma.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_brain_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_kidney_nondiseased.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lung_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lymph_node.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lymph_node_5k.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_pancreatic_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_breast_cancer.h5ad']
In [7]:
adata = sc.read_h5ad('/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lymph_node.h5ad')
In [8]:
from pathlib import Path
import spindle_dev
import spindle_dev.metrics as metrics
import spindle_dev.index as index
import spindle_dev.preprocessing as preprocessing
import spindle_dev.plotting as plotting
import spindle_dev.test as test
import spindle_dev.search as search
import spindle_dev.typing as typing
# Reload to pick up code changes without restarting the kernel
import time
import scanpy as sc
import glob
import pandas as pd
import numpy as np
from concurrent.futures import ThreadPoolExecutor, as_completed
from joblib import Parallel, delayed
import argparse
In [9]:
result_dir = "/data/sarkar_lab/Projects/spindle_dev/results/hlymph_10X/"
Path(result_dir).mkdir(parents=True, exist_ok=True)
In [10]:
all_genes = True
resolution=0.5
min_final_size=10
start_time = time.time()
coords = adata.obsm["spatial"]
tiles = preprocessing.build_quadtree_tiles(coords, max_pts=200, min_side=0.0, max_depth=40)
# remove tiles with less than 5 spots
tiles = [tile for tile in tiles if len(tile.idx) >= 5]
tiles = preprocessing.reindex_tiles(tiles)
num_genes = adata.n_vars
genes_work, gene_idx = spindle_dev.preprocessing.topvar_genes(adata, G=num_genes)
tile_covs = spindle_dev.preprocessing.build_tile_covs_full_serial(adata, tiles, gene_idx, eps=1e-6)
data = index.ProcessedData(tiles, tile_covs, genes_work, adata.n_obs)
if 'pca' not in data.latent:
data.reduce_dim(num_pca_components=30, n_components=2, do_umap=True)
data.cluster_spds(cluster_distance="tree", cluster_method="leiden", resolution=resolution)
data.assign_label_to_spots()
data.get_corr_mean_by_cluster()
out_dict = data.get_adaptive_runs(find_blocks=True, with_size_guard=True,min_final_size=min_final_size,max_final_size=100)
epsilon_block_wise_dict = {}
epsilon_dict = {}
for cluster_id in set(data.labels):
eps_per_block, eps_elbow_per_block, eps = index.choose_adaptive_epsilons(data, cluster_id, k_target_per_block=64)
epsilon_block_wise_dict[int(cluster_id)] = eps_elbow_per_block
epsilon_dict[int(cluster_id)] = eps
[2026-01-21 04:40:16,789] INFO spindle_dev.index: Clustering SPD-s using 'tree' distance. [2026-01-21 04:40:16,806] INFO spindle_dev.index: Building ultrametric features from SPD matrices. [2026-01-21 04:40:28,776] INFO spindle_dev.index: Computing latent features from the tree representations. [2026-01-21 04:40:29,851] INFO spindle_dev.index: Reducing latent features to 30 dimensions using PCA. [2026-01-21 04:40:43,140] INFO spindle_dev.index: Explained variance ratios by PCA components: [0.04082877 0.02090289 0.01214787 0.00703324 0.00602298 0.00583663 0.00484961 0.00420947 0.00341713 0.00326079 0.00293997 0.00285414 0.00278242 0.00267546 0.00262346 0.00254401 0.00253857 0.00249984 0.00243455 0.00240164 0.00238939 0.00234241 0.0022555 0.0022111 0.00219762 0.0021745 0.00212202 0.0020705 0.00205612 0.00198888] [2026-01-21 04:40:43,147] INFO spindle_dev.index: Reducing latent features to 2 dimensions using UMAP. /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/umap/umap_.py:1952: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism. warn( [2026-01-21 04:40:57,875] INFO spindle_dev.index: Clustering SPD-s using 'tree' distance. [2026-01-21 04:40:57,876] INFO spindle_dev.index: Clustering SPD matrices using Leiden clustering with resolution 0.50. [2026-01-21 04:40:58,262] INFO spindle_dev.index: Since clustering method is tree, I am going to find global order per cluster [2026-01-21 04:40:58,263] INFO spindle_dev.index: Finding consensus tree for cluster 0 [2026-01-21 04:40:58,483] INFO spindle_dev.index: Finding consensus tree for cluster 1 [2026-01-21 04:40:58,693] INFO spindle_dev.index: Finding consensus tree for cluster 2 [2026-01-21 04:40:58,894] INFO spindle_dev.index: Finding consensus tree for cluster 3 [2026-01-21 04:40:59,052] INFO spindle_dev.index: Finding consensus tree for cluster 4 [2026-01-21 04:40:59,152] INFO spindle_dev.index: Finding consensus tree for cluster 5 [2026-01-21 04:41:03,060] INFO spindle_dev.index: Computing mean correlation matrix for cluster 0 [2026-01-21 04:41:05,891] INFO spindle_dev.index: Computing mean correlation matrix for cluster 1 [2026-01-21 04:41:07,768] INFO spindle_dev.index: Computing mean correlation matrix for cluster 2 [2026-01-21 04:41:08,542] INFO spindle_dev.index: Computing mean correlation matrix for cluster 3 [2026-01-21 04:41:09,454] INFO spindle_dev.index: Computing mean correlation matrix for cluster 4 [2026-01-21 04:41:10,088] INFO spindle_dev.index: Computing mean correlation matrix for cluster 5 [2026-01-21 04:41:10,263] INFO spindle_dev.index: Finding adaptive block runs for cluster 0 [2026-01-21 04:41:10,667] INFO spindle_dev.index: Chose t=0.8953150000734291 resulting in 269 blocks instead of 234 blocks would have gotten by default [2026-01-21 04:41:10,720] INFO spindle_dev.index: Final block runs for cluster 0: 33 blocks. [2026-01-21 04:41:10,720] INFO spindle_dev.index: Finding adaptive block runs for cluster 1 [2026-01-21 04:41:11,108] INFO spindle_dev.index: Chose t=0.8974951970517207 resulting in 247 blocks instead of 219 blocks would have gotten by default [2026-01-21 04:41:11,155] INFO spindle_dev.index: Final block runs for cluster 1: 30 blocks. [2026-01-21 04:41:11,155] INFO spindle_dev.index: Finding adaptive block runs for cluster 2 [2026-01-21 04:41:11,545] INFO spindle_dev.index: Chose t=0.9153192440240208 resulting in 190 blocks instead of 182 blocks would have gotten by default [2026-01-21 04:41:11,575] INFO spindle_dev.index: Final block runs for cluster 2: 22 blocks. [2026-01-21 04:41:11,575] INFO spindle_dev.index: Finding adaptive block runs for cluster 3 [2026-01-21 04:41:11,965] INFO spindle_dev.index: Chose t=0.9106198620507007 resulting in 285 blocks instead of 267 blocks would have gotten by default [2026-01-21 04:41:12,028] INFO spindle_dev.index: Final block runs for cluster 3: 32 blocks. [2026-01-21 04:41:12,028] INFO spindle_dev.index: Finding adaptive block runs for cluster 4 [2026-01-21 04:41:12,417] INFO spindle_dev.index: Chose t=0.9374727137890102 resulting in 246 blocks instead of 291 blocks would have gotten by default [2026-01-21 04:41:12,467] INFO spindle_dev.index: Final block runs for cluster 4: 28 blocks. [2026-01-21 04:41:12,468] INFO spindle_dev.index: Finding adaptive block runs for cluster 5 [2026-01-21 04:41:12,858] INFO spindle_dev.index: Chose t=0.9281580392858046 resulting in 252 blocks instead of 271 blocks would have gotten by default [2026-01-21 04:41:12,906] INFO spindle_dev.index: Final block runs for cluster 5: 28 blocks.
In [11]:
import matplotlib.pylab as plt
from matplotlib.patches import Rectangle
from matplotlib.collections import PatchCollection
In [12]:
import numpy as np
palette = sc.pl.palettes.default_20
point_colors = [palette[lab] for lab in data.spot_label.values()]
labels = data.labels
indices = np.array([int(i) for i in data.spot_label.keys()])
coords = adata.obsm['spatial']
plt.scatter(coords[indices,0], coords[indices,1], s=1, c=point_colors, lw=0, alpha=0.8)
patches = []
for t in tiles:
bbox = t.bbox if hasattr(t, "bbox") else t["bbox"] # (xmin,ymin,xmax,ymax)
x0, y0, x1, y1 = bbox
patches.append(Rectangle((x0, y0), x1 - x0, y1 - y0, fill=False))
pc = PatchCollection(patches, match_original=True, linewidths=0.2, edgecolors='b', alpha=0.3)
# add collection
ax = plt.gca()
ax.add_collection(pc)
ax.invert_yaxis()
ax.axis('off')
plt.savefig(f"{result_dir}/spatial_plot_with_cluster.png", dpi=300)
plt.show()
In [13]:
# Create colors for clusters
import numpy as np
import matplotlib.pyplot as plt
n_clusters = len(set(data.labels))
# Use Scanpy's default_20 palette for consistency
palette = sc.pl.palettes.default_20
cluster_colors = [palette[lab] for lab in data.labels]
plt.scatter(data.latent['umap'][:,0], data.latent['umap'][:,1], s=6, lw=0, alpha=0.8, c=cluster_colors)
# Write label on the plot
# with backgrond circle for better visibility
# and bold font
for i in range(n_clusters):
cluster_points = data.latent['umap'][data.labels == i]
if len(cluster_points) == 0:
continue
x_mean = np.mean(cluster_points[:,0])
y_mean = np.mean(cluster_points[:,1])
plt.text(x_mean, y_mean, str(i), color='black', fontsize=10, ha='center', va='center', fontweight='bold',
bbox=dict(facecolor='white', edgecolor='black', boxstyle='circle,pad=0.5', alpha=0.7))
# take away top and right border
ax = plt.gca()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
# remove axis ticks
ax.set_xticks([])
ax.set_yticks([])
plt.savefig(f"{result_dir}/umap_plot_with_cluster_labels.png", dpi=300)
plt.show()
In [14]:
epsilon_block_wise_dict = {}
epsilon_dict = {}
for cluster_id in set(data.labels):
eps_per_block, eps_elbow_per_block, eps = index.choose_adaptive_epsilons(data, cluster_id, k_target_per_block=64)
epsilon_block_wise_dict[int(cluster_id)] = eps_elbow_per_block
epsilon_dict[int(cluster_id)] = eps
In [15]:
epsilon_dict
Out[15]:
{0: np.float64(5.6073310753416),
1: np.float64(5.957970216475703),
2: np.float64(8.066044544045555),
3: np.float64(7.942401914680533),
4: np.float64(5.564286986307886),
5: np.float64(4.676282788276763)}
In [16]:
config = typing.IndexConfig()
config.epsilon_dict = epsilon_dict
config.epsilon_block_wise_dict = epsilon_block_wise_dict
config.threshold_type = 'block-wise'
config.kmean_method = 'epsilon_net'
In [17]:
# Create index
dag_dict, stat, dist_list = index.index_spds(data, config=config)
[2026-01-21 04:41:38,984] INFO spindle_dev.index: Processing cluster 0 [2026-01-21 04:41:38,988] INFO spindle_dev.index: Building SPD index with epsilon=5.6073310753416 [2026-01-21 04:41:38,992] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:41:39,423] INFO spindle_dev.index: Cluster 0: 692 SPDs, 33 blocks [2026-01-21 04:41:39,426] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:41:39,535] INFO spindle_dev.index: Finished block 0 in 0.11 seconds, found 13 clusters. [2026-01-21 04:41:39,538] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:41:39,657] INFO spindle_dev.index: Finished block 1 in 0.12 seconds, found 15 clusters. [2026-01-21 04:41:39,660] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:41:39,774] INFO spindle_dev.index: Finished block 2 in 0.11 seconds, found 14 clusters. [2026-01-21 04:41:39,777] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:41:39,900] INFO spindle_dev.index: Finished block 3 in 0.12 seconds, found 16 clusters. [2026-01-21 04:41:39,904] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:41:40,019] INFO spindle_dev.index: Finished block 4 in 0.11 seconds, found 14 clusters. [2026-01-21 04:41:40,022] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:41:40,143] INFO spindle_dev.index: Finished block 5 in 0.12 seconds, found 15 clusters. [2026-01-21 04:41:40,145] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:41:40,277] INFO spindle_dev.index: Finished block 6 in 0.13 seconds, found 16 clusters. [2026-01-21 04:41:40,279] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:41:40,394] INFO spindle_dev.index: Finished block 7 in 0.12 seconds, found 14 clusters. [2026-01-21 04:41:40,396] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:41:40,511] INFO spindle_dev.index: Finished block 8 in 0.12 seconds, found 14 clusters. [2026-01-21 04:41:40,513] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:41:40,604] INFO spindle_dev.index: Finished block 9 in 0.09 seconds, found 10 clusters. [2026-01-21 04:41:40,606] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:41:40,714] INFO spindle_dev.index: Finished block 10 in 0.11 seconds, found 13 clusters. [2026-01-21 04:41:40,717] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:41:40,820] INFO spindle_dev.index: Finished block 11 in 0.10 seconds, found 12 clusters. [2026-01-21 04:41:40,824] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:41:40,912] INFO spindle_dev.index: Finished block 12 in 0.09 seconds, found 10 clusters. [2026-01-21 04:41:40,917] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:41:40,994] INFO spindle_dev.index: Finished block 13 in 0.08 seconds, found 8 clusters. [2026-01-21 04:41:40,996] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:41:41,076] INFO spindle_dev.index: Finished block 14 in 0.08 seconds, found 8 clusters. [2026-01-21 04:41:41,079] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:41:41,148] INFO spindle_dev.index: Finished block 15 in 0.07 seconds, found 6 clusters. [2026-01-21 04:41:41,150] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:41:41,197] INFO spindle_dev.index: Finished block 16 in 0.05 seconds, found 2 clusters. [2026-01-21 04:41:41,199] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:41:41,240] INFO spindle_dev.index: Finished block 17 in 0.04 seconds, found 1 clusters. [2026-01-21 04:41:41,242] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:41:41,299] INFO spindle_dev.index: Finished block 18 in 0.06 seconds, found 1 clusters. [2026-01-21 04:41:41,301] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:41:41,361] INFO spindle_dev.index: Finished block 19 in 0.06 seconds, found 2 clusters. [2026-01-21 04:41:41,365] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:41:41,429] INFO spindle_dev.index: Finished block 20 in 0.07 seconds, found 3 clusters. [2026-01-21 04:41:41,433] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:41:41,491] INFO spindle_dev.index: Finished block 21 in 0.06 seconds, found 2 clusters. [2026-01-21 04:41:41,495] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:41:41,537] INFO spindle_dev.index: Finished block 22 in 0.04 seconds, found 1 clusters. [2026-01-21 04:41:41,541] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:41:41,604] INFO spindle_dev.index: Finished block 23 in 0.06 seconds, found 3 clusters. [2026-01-21 04:41:41,607] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:41:41,650] INFO spindle_dev.index: Finished block 24 in 0.04 seconds, found 1 clusters. [2026-01-21 04:41:41,653] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:41:41,702] INFO spindle_dev.index: Finished block 25 in 0.05 seconds, found 1 clusters. [2026-01-21 04:41:41,705] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:41:41,745] INFO spindle_dev.index: Finished block 26 in 0.04 seconds, found 1 clusters. [2026-01-21 04:41:41,748] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:41:41,860] INFO spindle_dev.index: Finished block 27 in 0.11 seconds, found 8 clusters. [2026-01-21 04:41:41,862] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:41:41,922] INFO spindle_dev.index: Finished block 28 in 0.06 seconds, found 2 clusters. [2026-01-21 04:41:41,926] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:41:41,981] INFO spindle_dev.index: Finished block 29 in 0.06 seconds, found 3 clusters. [2026-01-21 04:41:41,983] INFO spindle_dev.index: Using epsilon-net clustering for block 30 [2026-01-21 04:41:42,128] INFO spindle_dev.index: Finished block 30 in 0.15 seconds, found 2 clusters. [2026-01-21 04:41:42,130] INFO spindle_dev.index: Using epsilon-net clustering for block 31 [2026-01-21 04:41:42,182] INFO spindle_dev.index: Finished block 31 in 0.05 seconds, found 2 clusters. [2026-01-21 04:41:42,185] INFO spindle_dev.index: Using epsilon-net clustering for block 32 [2026-01-21 04:41:42,253] INFO spindle_dev.index: Finished block 32 in 0.07 seconds, found 5 clusters. [2026-01-21 04:41:42,255] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:41:42,255] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:41:42,255] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:41:42,256] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:41:42,256] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:41:42,285] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:41:42,287] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:41:42,288] INFO spindle_dev.index: Processing cluster 1 [2026-01-21 04:41:42,288] INFO spindle_dev.index: Building SPD index with epsilon=5.957970216475703 [2026-01-21 04:41:42,288] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:41:42,606] INFO spindle_dev.index: Cluster 1: 688 SPDs, 30 blocks [2026-01-21 04:41:42,609] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:41:42,685] INFO spindle_dev.index: Finished block 0 in 0.08 seconds, found 7 clusters. [2026-01-21 04:41:42,688] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:41:42,771] INFO spindle_dev.index: Finished block 1 in 0.08 seconds, found 10 clusters. [2026-01-21 04:41:42,775] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:41:42,856] INFO spindle_dev.index: Finished block 2 in 0.08 seconds, found 9 clusters. [2026-01-21 04:41:42,858] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:41:42,970] INFO spindle_dev.index: Finished block 3 in 0.11 seconds, found 15 clusters. [2026-01-21 04:41:42,973] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:41:43,068] INFO spindle_dev.index: Finished block 4 in 0.10 seconds, found 11 clusters. [2026-01-21 04:41:43,070] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:41:43,177] INFO spindle_dev.index: Finished block 5 in 0.11 seconds, found 13 clusters. [2026-01-21 04:41:43,178] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:41:43,282] INFO spindle_dev.index: Finished block 6 in 0.10 seconds, found 12 clusters. [2026-01-21 04:41:43,286] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:41:43,390] INFO spindle_dev.index: Finished block 7 in 0.10 seconds, found 11 clusters. [2026-01-21 04:41:43,393] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:41:43,497] INFO spindle_dev.index: Finished block 8 in 0.10 seconds, found 11 clusters. [2026-01-21 04:41:43,500] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:41:43,603] INFO spindle_dev.index: Finished block 9 in 0.10 seconds, found 11 clusters. [2026-01-21 04:41:43,607] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:41:43,711] INFO spindle_dev.index: Finished block 10 in 0.10 seconds, found 11 clusters. [2026-01-21 04:41:43,715] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:41:43,804] INFO spindle_dev.index: Finished block 11 in 0.09 seconds, found 9 clusters. [2026-01-21 04:41:43,806] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:41:43,891] INFO spindle_dev.index: Finished block 12 in 0.09 seconds, found 10 clusters. [2026-01-21 04:41:43,893] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:41:43,966] INFO spindle_dev.index: Finished block 13 in 0.07 seconds, found 8 clusters. [2026-01-21 04:41:43,968] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:41:44,036] INFO spindle_dev.index: Finished block 14 in 0.07 seconds, found 7 clusters. [2026-01-21 04:41:44,038] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:41:44,091] INFO spindle_dev.index: Finished block 15 in 0.05 seconds, found 4 clusters. [2026-01-21 04:41:44,093] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:41:44,135] INFO spindle_dev.index: Finished block 16 in 0.04 seconds, found 2 clusters. [2026-01-21 04:41:44,137] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:41:44,184] INFO spindle_dev.index: Finished block 17 in 0.05 seconds, found 3 clusters. [2026-01-21 04:41:44,186] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:41:44,241] INFO spindle_dev.index: Finished block 18 in 0.06 seconds, found 4 clusters. [2026-01-21 04:41:44,243] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:41:44,305] INFO spindle_dev.index: Finished block 19 in 0.06 seconds, found 5 clusters. [2026-01-21 04:41:44,306] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:41:44,365] INFO spindle_dev.index: Finished block 20 in 0.06 seconds, found 4 clusters. [2026-01-21 04:41:44,367] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:41:44,420] INFO spindle_dev.index: Finished block 21 in 0.05 seconds, found 3 clusters. [2026-01-21 04:41:44,421] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:41:44,496] INFO spindle_dev.index: Finished block 22 in 0.07 seconds, found 2 clusters. [2026-01-21 04:41:44,498] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:41:44,588] INFO spindle_dev.index: Finished block 23 in 0.09 seconds, found 1 clusters. [2026-01-21 04:41:44,590] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:41:44,628] INFO spindle_dev.index: Finished block 24 in 0.04 seconds, found 1 clusters. [2026-01-21 04:41:44,630] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:41:44,716] INFO spindle_dev.index: Finished block 25 in 0.09 seconds, found 6 clusters. [2026-01-21 04:41:44,718] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:41:44,799] INFO spindle_dev.index: Finished block 26 in 0.08 seconds, found 9 clusters. [2026-01-21 04:41:44,801] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:41:44,860] INFO spindle_dev.index: Finished block 27 in 0.06 seconds, found 5 clusters. [2026-01-21 04:41:44,862] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:41:45,071] INFO spindle_dev.index: Finished block 28 in 0.21 seconds, found 7 clusters. [2026-01-21 04:41:45,072] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:41:45,142] INFO spindle_dev.index: Finished block 29 in 0.07 seconds, found 7 clusters. [2026-01-21 04:41:45,142] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:41:45,142] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:41:45,143] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:41:45,143] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:41:45,143] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:41:45,155] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:41:45,155] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:41:45,155] INFO spindle_dev.index: Processing cluster 2 [2026-01-21 04:41:45,156] INFO spindle_dev.index: Building SPD index with epsilon=8.066044544045555 [2026-01-21 04:41:45,156] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:41:45,194] INFO spindle_dev.index: Cluster 2: 538 SPDs, 22 blocks [2026-01-21 04:41:45,195] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:41:45,221] INFO spindle_dev.index: Finished block 0 in 0.03 seconds, found 2 clusters. [2026-01-21 04:41:45,222] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:41:45,246] INFO spindle_dev.index: Finished block 1 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:45,246] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:41:45,269] INFO spindle_dev.index: Finished block 2 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:45,270] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:41:45,293] INFO spindle_dev.index: Finished block 3 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:45,294] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:41:45,317] INFO spindle_dev.index: Finished block 4 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:45,318] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:41:45,342] INFO spindle_dev.index: Finished block 5 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:45,342] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:41:45,369] INFO spindle_dev.index: Finished block 6 in 0.03 seconds, found 2 clusters. [2026-01-21 04:41:45,369] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:41:45,393] INFO spindle_dev.index: Finished block 7 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:45,394] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:41:45,420] INFO spindle_dev.index: Finished block 8 in 0.03 seconds, found 2 clusters. [2026-01-21 04:41:45,420] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:41:45,446] INFO spindle_dev.index: Finished block 9 in 0.03 seconds, found 2 clusters. [2026-01-21 04:41:45,447] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:41:45,471] INFO spindle_dev.index: Finished block 10 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:45,472] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:41:45,502] INFO spindle_dev.index: Finished block 11 in 0.03 seconds, found 2 clusters. [2026-01-21 04:41:45,502] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:41:45,540] INFO spindle_dev.index: Finished block 12 in 0.04 seconds, found 4 clusters. [2026-01-21 04:41:45,541] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:41:45,570] INFO spindle_dev.index: Finished block 13 in 0.03 seconds, found 3 clusters. [2026-01-21 04:41:45,570] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:41:45,601] INFO spindle_dev.index: Finished block 14 in 0.03 seconds, found 3 clusters. [2026-01-21 04:41:45,601] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:41:45,638] INFO spindle_dev.index: Finished block 15 in 0.04 seconds, found 5 clusters. [2026-01-21 04:41:45,639] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:41:45,759] INFO spindle_dev.index: Finished block 16 in 0.12 seconds, found 14 clusters. [2026-01-21 04:41:45,760] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:41:45,939] INFO spindle_dev.index: Finished block 17 in 0.18 seconds, found 10 clusters. [2026-01-21 04:41:45,939] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:41:45,965] INFO spindle_dev.index: Finished block 18 in 0.03 seconds, found 2 clusters. [2026-01-21 04:41:45,965] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:41:46,164] INFO spindle_dev.index: Finished block 19 in 0.20 seconds, found 10 clusters. [2026-01-21 04:41:46,165] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:41:46,236] INFO spindle_dev.index: Finished block 20 in 0.07 seconds, found 5 clusters. [2026-01-21 04:41:46,236] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:41:46,332] INFO spindle_dev.index: Finished block 21 in 0.10 seconds, found 7 clusters. [2026-01-21 04:41:46,332] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:41:46,333] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:41:46,333] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:41:46,333] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:41:46,333] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:41:46,339] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:41:46,340] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:41:46,340] INFO spindle_dev.index: Processing cluster 3 [2026-01-21 04:41:46,340] INFO spindle_dev.index: Building SPD index with epsilon=7.942401914680533 [2026-01-21 04:41:46,340] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:41:46,369] INFO spindle_dev.index: Cluster 3: 425 SPDs, 32 blocks [2026-01-21 04:41:46,370] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:41:46,384] INFO spindle_dev.index: Finished block 0 in 0.01 seconds, found 1 clusters. [2026-01-21 04:41:46,385] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:41:46,398] INFO spindle_dev.index: Finished block 1 in 0.01 seconds, found 1 clusters. [2026-01-21 04:41:46,399] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:41:46,416] INFO spindle_dev.index: Finished block 2 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,416] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:41:46,431] INFO spindle_dev.index: Finished block 3 in 0.01 seconds, found 1 clusters. [2026-01-21 04:41:46,431] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:41:46,448] INFO spindle_dev.index: Finished block 4 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,449] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:41:46,468] INFO spindle_dev.index: Finished block 5 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,469] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:41:46,487] INFO spindle_dev.index: Finished block 6 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,488] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:41:46,509] INFO spindle_dev.index: Finished block 7 in 0.02 seconds, found 3 clusters. [2026-01-21 04:41:46,510] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:41:46,528] INFO spindle_dev.index: Finished block 8 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,528] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:41:46,547] INFO spindle_dev.index: Finished block 9 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,547] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:41:46,565] INFO spindle_dev.index: Finished block 10 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,566] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:41:46,584] INFO spindle_dev.index: Finished block 11 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,584] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:41:46,606] INFO spindle_dev.index: Finished block 12 in 0.02 seconds, found 3 clusters. [2026-01-21 04:41:46,606] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:41:46,628] INFO spindle_dev.index: Finished block 13 in 0.02 seconds, found 3 clusters. [2026-01-21 04:41:46,628] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:41:46,647] INFO spindle_dev.index: Finished block 14 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,648] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:41:46,667] INFO spindle_dev.index: Finished block 15 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,667] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:41:46,688] INFO spindle_dev.index: Finished block 16 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,689] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:41:46,711] INFO spindle_dev.index: Finished block 17 in 0.02 seconds, found 3 clusters. [2026-01-21 04:41:46,711] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:41:46,734] INFO spindle_dev.index: Finished block 18 in 0.02 seconds, found 3 clusters. [2026-01-21 04:41:46,734] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:41:46,753] INFO spindle_dev.index: Finished block 19 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:46,754] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:41:46,779] INFO spindle_dev.index: Finished block 20 in 0.03 seconds, found 4 clusters. [2026-01-21 04:41:46,780] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:41:46,809] INFO spindle_dev.index: Finished block 21 in 0.03 seconds, found 5 clusters. [2026-01-21 04:41:46,809] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:41:46,840] INFO spindle_dev.index: Finished block 22 in 0.03 seconds, found 4 clusters. [2026-01-21 04:41:46,840] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:41:46,869] INFO spindle_dev.index: Finished block 23 in 0.03 seconds, found 5 clusters. [2026-01-21 04:41:46,869] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:41:46,895] INFO spindle_dev.index: Finished block 24 in 0.03 seconds, found 4 clusters. [2026-01-21 04:41:46,896] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:41:46,918] INFO spindle_dev.index: Finished block 25 in 0.02 seconds, found 3 clusters. [2026-01-21 04:41:46,919] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:41:46,954] INFO spindle_dev.index: Finished block 26 in 0.04 seconds, found 7 clusters. [2026-01-21 04:41:46,955] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:41:46,990] INFO spindle_dev.index: Finished block 27 in 0.04 seconds, found 6 clusters. [2026-01-21 04:41:46,991] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:41:47,022] INFO spindle_dev.index: Finished block 28 in 0.03 seconds, found 5 clusters. [2026-01-21 04:41:47,023] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:41:47,053] INFO spindle_dev.index: Finished block 29 in 0.03 seconds, found 4 clusters. [2026-01-21 04:41:47,054] INFO spindle_dev.index: Using epsilon-net clustering for block 30 [2026-01-21 04:41:47,105] INFO spindle_dev.index: Finished block 30 in 0.05 seconds, found 8 clusters. [2026-01-21 04:41:47,106] INFO spindle_dev.index: Using epsilon-net clustering for block 31 [2026-01-21 04:41:47,201] INFO spindle_dev.index: Finished block 31 in 0.10 seconds, found 10 clusters. [2026-01-21 04:41:47,202] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:41:47,202] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:41:47,202] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:41:47,203] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:41:47,203] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:41:47,210] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:41:47,211] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:41:47,211] INFO spindle_dev.index: Processing cluster 4 [2026-01-21 04:41:47,211] INFO spindle_dev.index: Building SPD index with epsilon=5.564286986307886 [2026-01-21 04:41:47,211] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:41:47,234] INFO spindle_dev.index: Cluster 4: 340 SPDs, 28 blocks [2026-01-21 04:41:47,235] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:41:47,272] INFO spindle_dev.index: Finished block 0 in 0.04 seconds, found 10 clusters. [2026-01-21 04:41:47,273] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:41:47,318] INFO spindle_dev.index: Finished block 1 in 0.05 seconds, found 14 clusters. [2026-01-21 04:41:47,319] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:41:47,362] INFO spindle_dev.index: Finished block 2 in 0.04 seconds, found 13 clusters. [2026-01-21 04:41:47,363] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:41:47,411] INFO spindle_dev.index: Finished block 3 in 0.05 seconds, found 15 clusters. [2026-01-21 04:41:47,412] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:41:47,458] INFO spindle_dev.index: Finished block 4 in 0.05 seconds, found 14 clusters. [2026-01-21 04:41:47,459] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:41:47,502] INFO spindle_dev.index: Finished block 5 in 0.04 seconds, found 13 clusters. [2026-01-21 04:41:47,503] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:41:47,552] INFO spindle_dev.index: Finished block 6 in 0.05 seconds, found 15 clusters. [2026-01-21 04:41:47,552] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:41:47,594] INFO spindle_dev.index: Finished block 7 in 0.04 seconds, found 12 clusters. [2026-01-21 04:41:47,594] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:41:47,633] INFO spindle_dev.index: Finished block 8 in 0.04 seconds, found 11 clusters. [2026-01-21 04:41:47,634] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:41:47,667] INFO spindle_dev.index: Finished block 9 in 0.03 seconds, found 9 clusters. [2026-01-21 04:41:47,667] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:41:47,708] INFO spindle_dev.index: Finished block 10 in 0.04 seconds, found 12 clusters. [2026-01-21 04:41:47,709] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:41:47,753] INFO spindle_dev.index: Finished block 11 in 0.04 seconds, found 13 clusters. [2026-01-21 04:41:47,754] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:41:47,778] INFO spindle_dev.index: Finished block 12 in 0.02 seconds, found 5 clusters. [2026-01-21 04:41:47,778] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:41:47,810] INFO spindle_dev.index: Finished block 13 in 0.03 seconds, found 8 clusters. [2026-01-21 04:41:47,810] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:41:47,837] INFO spindle_dev.index: Finished block 14 in 0.03 seconds, found 6 clusters. [2026-01-21 04:41:47,837] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:41:47,864] INFO spindle_dev.index: Finished block 15 in 0.03 seconds, found 4 clusters. [2026-01-21 04:41:47,864] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:41:47,880] INFO spindle_dev.index: Finished block 16 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:47,881] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:41:47,902] INFO spindle_dev.index: Finished block 17 in 0.02 seconds, found 4 clusters. [2026-01-21 04:41:47,902] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:41:47,920] INFO spindle_dev.index: Finished block 18 in 0.02 seconds, found 2 clusters. [2026-01-21 04:41:47,921] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:41:47,944] INFO spindle_dev.index: Finished block 19 in 0.02 seconds, found 3 clusters. [2026-01-21 04:41:47,945] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:41:47,971] INFO spindle_dev.index: Finished block 20 in 0.03 seconds, found 4 clusters. [2026-01-21 04:41:47,972] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:41:47,991] INFO spindle_dev.index: Finished block 21 in 0.02 seconds, found 3 clusters. [2026-01-21 04:41:47,991] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:41:48,012] INFO spindle_dev.index: Finished block 22 in 0.02 seconds, found 3 clusters. [2026-01-21 04:41:48,013] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:41:48,032] INFO spindle_dev.index: Finished block 23 in 0.02 seconds, found 3 clusters. [2026-01-21 04:41:48,033] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:41:48,066] INFO spindle_dev.index: Finished block 24 in 0.03 seconds, found 2 clusters. [2026-01-21 04:41:48,067] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:41:48,086] INFO spindle_dev.index: Finished block 25 in 0.02 seconds, found 3 clusters. [2026-01-21 04:41:48,086] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:41:48,151] INFO spindle_dev.index: Finished block 26 in 0.07 seconds, found 2 clusters. [2026-01-21 04:41:48,152] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:41:48,205] INFO spindle_dev.index: Finished block 27 in 0.05 seconds, found 2 clusters. [2026-01-21 04:41:48,205] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:41:48,206] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:41:48,206] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:41:48,206] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:41:48,206] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:41:48,212] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:41:48,213] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:41:48,213] INFO spindle_dev.index: Processing cluster 5 [2026-01-21 04:41:48,213] INFO spindle_dev.index: Building SPD index with epsilon=4.676282788276763 [2026-01-21 04:41:48,213] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:41:48,221] INFO spindle_dev.index: Cluster 5: 89 SPDs, 28 blocks [2026-01-21 04:41:48,222] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:41:48,235] INFO spindle_dev.index: Finished block 0 in 0.01 seconds, found 13 clusters. [2026-01-21 04:41:48,235] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:41:48,255] INFO spindle_dev.index: Finished block 1 in 0.02 seconds, found 25 clusters. [2026-01-21 04:41:48,255] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:41:48,278] INFO spindle_dev.index: Finished block 2 in 0.02 seconds, found 29 clusters. [2026-01-21 04:41:48,278] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:41:48,299] INFO spindle_dev.index: Finished block 3 in 0.02 seconds, found 26 clusters. [2026-01-21 04:41:48,299] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:41:48,321] INFO spindle_dev.index: Finished block 4 in 0.02 seconds, found 28 clusters. [2026-01-21 04:41:48,321] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:41:48,342] INFO spindle_dev.index: Finished block 5 in 0.02 seconds, found 26 clusters. [2026-01-21 04:41:48,342] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:41:48,361] INFO spindle_dev.index: Finished block 6 in 0.02 seconds, found 24 clusters. [2026-01-21 04:41:48,362] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:41:48,379] INFO spindle_dev.index: Finished block 7 in 0.02 seconds, found 21 clusters. [2026-01-21 04:41:48,379] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:41:48,392] INFO spindle_dev.index: Finished block 8 in 0.01 seconds, found 15 clusters. [2026-01-21 04:41:48,393] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:41:48,407] INFO spindle_dev.index: Finished block 9 in 0.01 seconds, found 17 clusters. [2026-01-21 04:41:48,408] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:41:48,419] INFO spindle_dev.index: Finished block 10 in 0.01 seconds, found 12 clusters. [2026-01-21 04:41:48,419] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:41:48,430] INFO spindle_dev.index: Finished block 11 in 0.01 seconds, found 11 clusters. [2026-01-21 04:41:48,430] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:41:48,441] INFO spindle_dev.index: Finished block 12 in 0.01 seconds, found 12 clusters. [2026-01-21 04:41:48,441] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:41:48,453] INFO spindle_dev.index: Finished block 13 in 0.01 seconds, found 12 clusters. [2026-01-21 04:41:48,453] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:41:48,462] INFO spindle_dev.index: Finished block 14 in 0.01 seconds, found 9 clusters. [2026-01-21 04:41:48,463] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:41:48,469] INFO spindle_dev.index: Finished block 15 in 0.01 seconds, found 5 clusters. [2026-01-21 04:41:48,469] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:41:48,479] INFO spindle_dev.index: Finished block 16 in 0.01 seconds, found 10 clusters. [2026-01-21 04:41:48,480] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:41:48,489] INFO spindle_dev.index: Finished block 17 in 0.01 seconds, found 8 clusters. [2026-01-21 04:41:48,489] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:41:48,498] INFO spindle_dev.index: Finished block 18 in 0.01 seconds, found 9 clusters. [2026-01-21 04:41:48,498] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:41:48,505] INFO spindle_dev.index: Finished block 19 in 0.01 seconds, found 5 clusters. [2026-01-21 04:41:48,506] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:41:48,514] INFO spindle_dev.index: Finished block 20 in 0.01 seconds, found 7 clusters. [2026-01-21 04:41:48,514] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:41:48,530] INFO spindle_dev.index: Finished block 21 in 0.02 seconds, found 6 clusters. [2026-01-21 04:41:48,530] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:41:48,541] INFO spindle_dev.index: Finished block 22 in 0.01 seconds, found 4 clusters. [2026-01-21 04:41:48,541] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:41:48,545] INFO spindle_dev.index: Finished block 23 in 0.00 seconds, found 1 clusters. [2026-01-21 04:41:48,546] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:41:48,552] INFO spindle_dev.index: Finished block 24 in 0.01 seconds, found 3 clusters. [2026-01-21 04:41:48,553] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:41:48,557] INFO spindle_dev.index: Finished block 25 in 0.00 seconds, found 2 clusters. [2026-01-21 04:41:48,558] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:41:48,589] INFO spindle_dev.index: Finished block 26 in 0.03 seconds, found 3 clusters. [2026-01-21 04:41:48,590] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:41:48,595] INFO spindle_dev.index: Finished block 27 in 0.01 seconds, found 2 clusters. [2026-01-21 04:41:48,595] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:41:48,595] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:41:48,596] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:41:48,596] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:41:48,596] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:41:48,599] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:41:48,599] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances.
In [18]:
search_cfg = search.SearchConfig(max_results=2, debug=False, max_failed_starts=5, max_failed_paths=10)
In [19]:
test_results = test.run_sanity_search(data, dag_dict, config, search_cfg, max_queries=2000, skip_baseline=True)
[2026-01-21 04:41:49,276] INFO spindle_dev.test: Built ground-truth paths for 692 SPDs in cluster 0 across 33 blocks. [2026-01-21 04:41:49,838] INFO spindle_dev.test: Built ground-truth paths for 688 SPDs in cluster 1 across 30 blocks. [2026-01-21 04:41:50,077] INFO spindle_dev.test: Built ground-truth paths for 538 SPDs in cluster 2 across 22 blocks. [2026-01-21 04:41:50,320] INFO spindle_dev.test: Built ground-truth paths for 425 SPDs in cluster 3 across 32 blocks. [2026-01-21 04:41:50,447] INFO spindle_dev.test: Built ground-truth paths for 340 SPDs in cluster 4 across 28 blocks. [2026-01-21 04:41:50,461] INFO spindle_dev.test: Built ground-truth paths for 89 SPDs in cluster 5 across 28 blocks. [2026-01-21 04:41:50,462] INFO spindle_dev.test: Created ground-truth paths for all clusters. [2026-01-21 04:41:50,462] INFO spindle_dev.test: Search config: SearchConfig(max_results=2, max_failed_starts=5, max_failed_paths=10, total_paths_limit=1000, deterministic=DeterministicConfig(seed=0), debug=False) [2026-01-21 04:41:50,463] INFO spindle_dev.test: Running sanity search with 2000 queries. 100%|██████████| 2000/2000 [00:15<00:00, 132.18it/s] [2026-01-21 04:42:05,597] INFO spindle_dev.test: Sanity search: 2000 queries, 2000 exact path matches, 2000 leaf matches, mean search_time=0.0065s
In [20]:
fig = plotting.visualize_block_dag_sankey_scaled(dag_dict[2].nodes, height=800, thickness=20.0)
fig
In [21]:
import pandas as pd
def get_dag_stats(dag_dict):
dag_stats = []
for cluster_id, cluster_dag in dag_dict.items():
block_to_node = cluster_dag.block_to_node_indices
for node in cluster_dag.nodes:
num_spds = len(node.metadata.members)
block_size = node.metadata.mean.shape[0]
block_id = node.block_index
radius = node.metadata.radius
#print(f"Cluster_id {cluster_id} Node id {node.global_node_id} Block {block_id}: {num_spds} spds")
dag_stats.append({
"cluster_id": cluster_id,
"node_id": node.global_node_id,
"block_id": block_id,
"block_cluster_id": node.block_cluster_id,
"num_spds": num_spds,
"radius": radius,
"block_size": block_size
})
dag_stats_df = pd.DataFrame(dag_stats)
return dag_stats_df
In [22]:
dag_stats_df = get_dag_stats(dag_dict)
In [23]:
score_list = []
for cluster_id in set(data.labels):
cluster_dag = dag_dict[cluster_id]
cluster_score = index.score_nodes_from_a_cluster(data, cluster_id, dag_dict, take_representative_mean=True)
score_list.extend(cluster_score)
100%|██████████| 238/238 [00:00<00:00, 964.37it/s] 100%|██████████| 218/218 [00:00<00:00, 664.87it/s] 100%|██████████| 87/87 [00:00<00:00, 131.96it/s] 100%|██████████| 107/107 [00:00<00:00, 494.27it/s] 100%|██████████| 207/207 [00:00<00:00, 765.81it/s] 100%|██████████| 345/345 [00:00<00:00, 734.05it/s]
In [24]:
node_scores = []
for node_score in score_list:
node_scores.append({
"cluster_id": node_score["cluster_id"],
"block_id": node_score["block_id"],
"node_id": node_score["node_id"],
"node_score": node_score["node_score"],
"num_spds": node_score["num_spds"],
"block_size": node_score["block_size"],
"E": node_score["E"],
"Q": node_score["Q"],
"S": node_score["S"],
"num_modules": len(node_score["modules"])
})
node_score_df = pd.DataFrame(node_scores)
In [25]:
node_score_df.sort_values(by='node_score', ascending=False).head(20)
Out[25]:
| cluster_id | block_id | node_id | node_score | num_spds | block_size | E | Q | S | num_modules | |
|---|---|---|---|---|---|---|---|---|---|---|
| 513 | 2 | 17 | 57 | 0.243729 | 2 | 48 | 2.259945 | 0.191086 | 0.435612 | 3 |
| 530 | 2 | 19 | 74 | 0.232365 | 2 | 52 | 2.365266 | 0.177967 | 0.447987 | 5 |
| 526 | 2 | 19 | 70 | 0.212208 | 1 | 52 | 2.054459 | 0.190328 | 0.457301 | 5 |
| 1147 | 5 | 16 | 290 | 0.204470 | 1 | 10 | 0.655841 | 0.540124 | 0.422787 | 3 |
| 525 | 2 | 19 | 69 | 0.202023 | 2 | 52 | 1.848070 | 0.182815 | 0.402044 | 4 |
| 538 | 2 | 21 | 82 | 0.198203 | 2 | 32 | 2.137909 | 0.184733 | 0.498151 | 2 |
| 498 | 2 | 16 | 42 | 0.188858 | 1 | 30 | 2.499646 | 0.173551 | 0.564661 | 2 |
| 510 | 2 | 17 | 54 | 0.187747 | 14 | 48 | 2.531355 | 0.150079 | 0.505807 | 5 |
| 502 | 2 | 16 | 46 | 0.179420 | 2 | 30 | 2.479064 | 0.167538 | 0.568017 | 3 |
| 505 | 2 | 16 | 49 | 0.177893 | 1 | 30 | 1.950891 | 0.184218 | 0.505013 | 4 |
| 523 | 2 | 19 | 67 | 0.174896 | 3 | 52 | 1.659023 | 0.171488 | 0.385261 | 6 |
| 511 | 2 | 17 | 55 | 0.169813 | 5 | 48 | 1.551361 | 0.179338 | 0.389644 | 5 |
| 164 | 0 | 11 | 164 | 0.166171 | 8 | 10 | 0.634193 | 0.505121 | 0.481275 | 4 |
| 527 | 2 | 19 | 71 | 0.164733 | 6 | 52 | 1.598675 | 0.170321 | 0.395007 | 5 |
| 534 | 2 | 20 | 78 | 0.161301 | 1 | 26 | 1.943349 | 0.182406 | 0.544967 | 2 |
| 529 | 2 | 19 | 73 | 0.161013 | 4 | 52 | 1.766246 | 0.164506 | 0.445854 | 7 |
| 528 | 2 | 19 | 72 | 0.159444 | 2 | 52 | 1.466239 | 0.162362 | 0.330244 | 5 |
| 514 | 2 | 17 | 58 | 0.157999 | 2 | 48 | 2.580272 | 0.123173 | 0.502867 | 3 |
| 1032 | 5 | 7 | 175 | 0.154098 | 2 | 10 | 0.560713 | 0.533141 | 0.484519 | 4 |
| 533 | 2 | 20 | 77 | 0.153934 | 2 | 26 | 2.081050 | 0.157143 | 0.529288 | 4 |
In [26]:
importlib.reload(plotting)
Out[26]:
<module 'spindle_dev.plotting' from '/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/plotting.py'>
In [27]:
id = 513
scores_846, fig_846 = plotting.plot_module_heatmap_plus_spatial(
id,
score_list,
node_score_df,
dag_stats_df,
dag_dict,
data,
adata,
spot_size=0.2
)
Creating combined figure for cluster 2, node 57, block 17
In [28]:
fig_846.savefig(f"{result_dir}/{id}_module_0_1_enrichment.png", dpi=300)
In [29]:
from spindle_dev import go_score
In [30]:
out_846 = go_score.enrich_modules_with_gseapy(score_list[id]['modules'])
Module 0: enrichment completed with 557 terms Module 1: enrichment completed with 672 terms Module 2: skipped (only 1 genes)
In [31]:
for module_id, module_genes, res in out_846:
if module_id == 0:
print(module_genes)
fig, axes = go_score.plot_module_enrichment_libraries(
module_id=module_id,
module_genes=module_genes,
res=res,
top_n=7,
ncols=2,
bar_color="#D41515EE", # pick your color
tick_fontsize=10,
label_fontsize=10,
title_fontsize=8,
compact_height_per_term=0.2,
min_fig_h=6,
save_dir=f"{result_dir}/go_enrichment/enrichment_{id}"
)
if fig is not None:
plt.show()
['APOLD1', 'CAVIN1', 'CAVIN2', 'CD93', 'CXCL2', 'ECSCR', 'EGFL7', 'FBN1', 'LYVE1', 'MALL', 'MARCO', 'MYC', 'NPDC1', 'PDPN', 'PECAM1', 'PROX1', 'RAMP2', 'RBP5', 'SERPINB9', 'SH2D3C', 'SMYD2', 'STC2', 'TFPI', 'TM4SF18', 'TNC'] ['GO_Biological_Process_2023', 'KEGG_2021_Human', 'MSigDB_Hallmark_2020', 'Reactome_2022']
In [36]:
for module_id, module_genes, res in out_846:
if module_id == 0:
print(module_genes)
fig, axes = go_score.plot_module_enrichment_libraries_2(
module_id=module_id,
module_genes=module_genes,
res=res,
top_n=7,
ncols=2,
bar_color="#D41515EE", # pick your color
tick_fontsize=10,
label_fontsize=10,
title_fontsize=8,
compact_height_per_term=0.2,
min_fig_h=8,
plot_kind="dot",
size_range=(20, 200),
use_constrained_layout=False,
save_dir=f"{result_dir}/go_enrichment/enrichment_{id}"
)
if fig is not None:
plt.show()
['APOLD1', 'CAVIN1', 'CAVIN2', 'CD93', 'CXCL2', 'ECSCR', 'EGFL7', 'FBN1', 'LYVE1', 'MALL', 'MARCO', 'MYC', 'NPDC1', 'PDPN', 'PECAM1', 'PROX1', 'RAMP2', 'RBP5', 'SERPINB9', 'SH2D3C', 'SMYD2', 'STC2', 'TFPI', 'TM4SF18', 'TNC']
/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/go_score.py:519: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
In [32]:
# Create a dictionary of tile_id to block_id
# Clusters do not share tile ids across them
from collections import defaultdict
tile_to_block_dict = {}
cluster_to_tile = defaultdict(set)
for cluster_id, cluster_dag in dag_dict.items():
block_to_node = cluster_dag.block_to_node_indices
for node in cluster_dag.nodes:
member_tile_ids = [tid for tid,_ in node.metadata.members]
for tile_id in member_tile_ids:
if not tile_id in tile_to_block_dict:
tile_to_block_dict[tile_id] = {}
tile_to_block_dict[tile_id][node.block_index] = node.block_cluster_id
cluster_to_tile[cluster_id].add(tile_id)
In [33]:
def get_spot_score_dict(block_id, cluster_id, tile_to_block_dict, grid=False):
spot_score = {}
tiles_of_cluster = cluster_to_tile[cluster_id]
for tid in tiles_of_cluster:
if tid not in tile_to_block_dict:
continue
c = tile_to_block_dict[tid][block_id]
t = data.metadata['tiles'][tid]
for id in t.idx:
spot_score[id] = c
fig, ax = plotting.plot_spot_module_scores(spot_score, tiles, adata, color='discrete', grid=grid)
return fig, ax
In [34]:
fig, ax = get_spot_score_dict(5, 0, tile_to_block_dict, grid=False)
In [35]:
sc.pl.spatial(adata,color=['PROX1', 'LYVE1'], spot_size=15, cmap='Reds')
/tmp/ipykernel_824584/2142307267.py:1: FutureWarning: Use `squidpy.pl.spatial_scatter` instead.
In [ ]: