In [1]:
# Set up sys.path so that 'src/spindle_dev' is importable as 'spindle_dev'
import sys
import importlib
from pathlib import Path
project_root = '/data/sarkar_lab/Projects/spindle_dev'
src_path = Path(project_root) / 'src'
if str(src_path) not in sys.path:
sys.path.insert(0, str(src_path))
import spindle_dev
# Reload to pick up code changes without restarting the kernel
importlib.reload(spindle_dev)
Out[1]:
<module 'spindle_dev' from '/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/__init__.py'>
In [2]:
import scanpy as sc
In [3]:
import glob
In [4]:
h5ad_files = glob.glob("/data/sarkar_lab/insitupy_demo_data_xenium/*.h5ad")
In [5]:
h5ad_files
Out[5]:
['/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_skin_melanoma.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_brain_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_kidney_nondiseased.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lung_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lymph_node.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lymph_node_5k.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_pancreatic_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_breast_cancer.h5ad']
In [6]:
adata = sc.read_h5ad('/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_skin_melanoma.h5ad')
In [8]:
from pathlib import Path
import spindle_dev
import spindle_dev.metrics as metrics
import spindle_dev.index as index
import spindle_dev.preprocessing as preprocessing
import spindle_dev.plotting as plotting
import spindle_dev.test as test
import spindle_dev.search as search
import spindle_dev.typing as typing
# Reload to pick up code changes without restarting the kernel
import time
import scanpy as sc
import glob
import pandas as pd
import numpy as np
from concurrent.futures import ThreadPoolExecutor, as_completed
from joblib import Parallel, delayed
import argparse
In [12]:
result_dir = "/data/sarkar_lab/Projects/spindle_dev/results/hskin_10X/"
Path(result_dir).mkdir(parents=True, exist_ok=True)
In [10]:
all_genes = True
resolution=0.5
min_final_size=10
start_time = time.time()
coords = adata.obsm["spatial"]
tiles = preprocessing.build_quadtree_tiles(coords, max_pts=200, min_side=0.0, max_depth=40)
# remove tiles with less than 5 spots
tiles = [tile for tile in tiles if len(tile.idx) >= 5]
tiles = preprocessing.reindex_tiles(tiles)
num_genes = adata.n_vars
genes_work, gene_idx = spindle_dev.preprocessing.topvar_genes(adata, G=num_genes)
tile_covs = spindle_dev.preprocessing.build_tile_covs_full_serial(adata, tiles, gene_idx, eps=1e-6)
data = index.ProcessedData(tiles, tile_covs, genes_work, adata.n_obs)
if 'pca' not in data.latent:
data.reduce_dim(num_pca_components=30, n_components=2, do_umap=True)
data.cluster_spds(cluster_distance="tree", cluster_method="leiden", resolution=resolution)
data.assign_label_to_spots()
data.get_corr_mean_by_cluster()
out_dict = data.get_adaptive_runs(find_blocks=True, with_size_guard=True,min_final_size=min_final_size,max_final_size=100)
epsilon_block_wise_dict = {}
epsilon_dict = {}
for cluster_id in set(data.labels):
eps_per_block, eps_elbow_per_block, eps = index.choose_adaptive_epsilons(data, cluster_id, k_target_per_block=64)
epsilon_block_wise_dict[int(cluster_id)] = eps_elbow_per_block
epsilon_dict[int(cluster_id)] = eps
[2026-01-19 22:29:18,281] INFO spindle_dev.index: Clustering SPD-s using 'tree' distance. [2026-01-19 22:29:18,370] INFO spindle_dev.index: Building ultrametric features from SPD matrices. [2026-01-19 22:29:26,279] INFO spindle_dev.index: Computing latent features from the tree representations. [2026-01-19 22:29:26,671] INFO spindle_dev.index: Reducing latent features to 30 dimensions using PCA. [2026-01-19 22:29:31,185] INFO spindle_dev.index: Explained variance ratios by PCA components: [0.0594365 0.04257701 0.02694363 0.02286748 0.01322526 0.00847291 0.00765662 0.00680464 0.00621679 0.00598492 0.0054304 0.00475169 0.00459882 0.00427495 0.00419172 0.00405914 0.00396039 0.00382956 0.00374104 0.00359517 0.00349659 0.00337946 0.00327844 0.00323106 0.00320595 0.00311978 0.00309778 0.00301003 0.00297968 0.00291783] [2026-01-19 22:29:31,186] INFO spindle_dev.index: Reducing latent features to 2 dimensions using UMAP. /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/umap/umap_.py:1952: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism. warn( [2026-01-19 22:29:40,117] INFO spindle_dev.index: Clustering SPD-s using 'tree' distance. [2026-01-19 22:29:40,118] INFO spindle_dev.index: Clustering SPD matrices using Leiden clustering with resolution 0.50. [2026-01-19 22:29:40,204] INFO spindle_dev.index: Since clustering method is tree, I am going to find global order per cluster [2026-01-19 22:29:40,205] INFO spindle_dev.index: Finding consensus tree for cluster 0 [2026-01-19 22:29:40,258] INFO spindle_dev.index: Finding consensus tree for cluster 1 [2026-01-19 22:29:40,307] INFO spindle_dev.index: Finding consensus tree for cluster 2 [2026-01-19 22:29:40,350] INFO spindle_dev.index: Finding consensus tree for cluster 3 [2026-01-19 22:29:40,382] INFO spindle_dev.index: Finding consensus tree for cluster 4 [2026-01-19 22:29:40,410] INFO spindle_dev.index: Finding consensus tree for cluster 5 [2026-01-19 22:29:40,688] INFO spindle_dev.index: Computing mean correlation matrix for cluster 0 [2026-01-19 22:29:41,174] INFO spindle_dev.index: Computing mean correlation matrix for cluster 1 [2026-01-19 22:29:41,622] INFO spindle_dev.index: Computing mean correlation matrix for cluster 2 [2026-01-19 22:29:41,984] INFO spindle_dev.index: Computing mean correlation matrix for cluster 3 [2026-01-19 22:29:42,151] INFO spindle_dev.index: Computing mean correlation matrix for cluster 4 [2026-01-19 22:29:42,301] INFO spindle_dev.index: Computing mean correlation matrix for cluster 5 [2026-01-19 22:29:42,457] INFO spindle_dev.index: Finding adaptive block runs for cluster 0 [2026-01-19 22:29:42,880] INFO spindle_dev.index: Chose t=0.8403484674528324 resulting in 250 blocks instead of 135 blocks would have gotten by default [2026-01-19 22:29:42,926] INFO spindle_dev.index: Final block runs for cluster 0: 28 blocks. [2026-01-19 22:29:42,926] INFO spindle_dev.index: Finding adaptive block runs for cluster 1 [2026-01-19 22:29:43,324] INFO spindle_dev.index: Chose t=0.84045364173602 resulting in 249 blocks instead of 110 blocks would have gotten by default [2026-01-19 22:29:43,370] INFO spindle_dev.index: Final block runs for cluster 1: 30 blocks. [2026-01-19 22:29:43,371] INFO spindle_dev.index: Finding adaptive block runs for cluster 2 [2026-01-19 22:29:43,769] INFO spindle_dev.index: Chose t=0.882815137034527 resulting in 209 blocks instead of 110 blocks would have gotten by default [2026-01-19 22:29:43,803] INFO spindle_dev.index: Final block runs for cluster 2: 26 blocks. [2026-01-19 22:29:43,804] INFO spindle_dev.index: Finding adaptive block runs for cluster 3 [2026-01-19 22:29:44,199] INFO spindle_dev.index: Chose t=0.8928216387309484 resulting in 213 blocks instead of 142 blocks would have gotten by default [2026-01-19 22:29:44,235] INFO spindle_dev.index: Final block runs for cluster 3: 26 blocks. [2026-01-19 22:29:44,236] INFO spindle_dev.index: Finding adaptive block runs for cluster 4 [2026-01-19 22:29:44,633] INFO spindle_dev.index: Chose t=0.8501217149922651 resulting in 196 blocks instead of 76 blocks would have gotten by default [2026-01-19 22:29:44,662] INFO spindle_dev.index: Final block runs for cluster 4: 24 blocks. [2026-01-19 22:29:44,663] INFO spindle_dev.index: Finding adaptive block runs for cluster 5 [2026-01-19 22:29:45,058] INFO spindle_dev.index: Chose t=0.869744973970191 resulting in 218 blocks instead of 98 blocks would have gotten by default [2026-01-19 22:29:45,093] INFO spindle_dev.index: Final block runs for cluster 5: 28 blocks.
In [11]:
import matplotlib.pylab as plt
from matplotlib.patches import Rectangle
from matplotlib.collections import PatchCollection
In [13]:
import numpy as np
palette = sc.pl.palettes.default_20
point_colors = [palette[lab] for lab in data.spot_label.values()]
labels = data.labels
indices = np.array([int(i) for i in data.spot_label.keys()])
coords = adata.obsm['spatial']
plt.scatter(coords[indices,0], coords[indices,1], s=1, c=point_colors, lw=0, alpha=0.8)
patches = []
for t in tiles:
bbox = t.bbox if hasattr(t, "bbox") else t["bbox"] # (xmin,ymin,xmax,ymax)
x0, y0, x1, y1 = bbox
patches.append(Rectangle((x0, y0), x1 - x0, y1 - y0, fill=False))
pc = PatchCollection(patches, match_original=True, linewidths=0.2, edgecolors='b', alpha=0.3)
# add collection
ax = plt.gca()
ax.add_collection(pc)
ax.invert_yaxis()
ax.axis('off')
plt.savefig(f"{result_dir}/spatial_plot_with_cluster.png", dpi=300)
plt.show()
In [14]:
# Create colors for clusters
import numpy as np
import matplotlib.pyplot as plt
n_clusters = len(set(data.labels))
# Use Scanpy's default_20 palette for consistency
palette = sc.pl.palettes.default_20
cluster_colors = [palette[lab] for lab in data.labels]
plt.scatter(data.latent['umap'][:,0], data.latent['umap'][:,1], s=6, lw=0, alpha=0.8, c=cluster_colors)
# Write label on the plot
# with backgrond circle for better visibility
# and bold font
for i in range(n_clusters):
cluster_points = data.latent['umap'][data.labels == i]
if len(cluster_points) == 0:
continue
x_mean = np.mean(cluster_points[:,0])
y_mean = np.mean(cluster_points[:,1])
plt.text(x_mean, y_mean, str(i), color='black', fontsize=10, ha='center', va='center', fontweight='bold',
bbox=dict(facecolor='white', edgecolor='black', boxstyle='circle,pad=0.5', alpha=0.7))
# take away top and right border
ax = plt.gca()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
# remove axis ticks
ax.set_xticks([])
ax.set_yticks([])
plt.savefig(f"{result_dir}/umap_plot_with_cluster_labels.png", dpi=300)
plt.show()
In [15]:
epsilon_block_wise_dict = {}
epsilon_dict = {}
for cluster_id in set(data.labels):
eps_per_block, eps_elbow_per_block, eps = index.choose_adaptive_epsilons(data, cluster_id, k_target_per_block=64)
epsilon_block_wise_dict[int(cluster_id)] = eps_elbow_per_block
epsilon_dict[int(cluster_id)] = eps
In [16]:
epsilon_dict
Out[16]:
{0: np.float64(5.552291233651296),
1: np.float64(6.528217907292994),
2: np.float64(6.086265230593296),
3: np.float64(5.8950403735406995),
4: np.float64(8.06822025162069),
5: np.float64(7.23115036152569)}
In [17]:
config = typing.IndexConfig()
config.epsilon_dict = epsilon_dict
config.epsilon_block_wise_dict = epsilon_block_wise_dict
config.threshold_type = 'constant'
config.kmean_method = 'epsilon_net'
In [18]:
# Create index
dag_dict, stat, dist_list = index.index_spds(data, config=config)
[2026-01-19 22:34:28,485] INFO spindle_dev.index: Processing cluster 0 [2026-01-19 22:34:28,486] INFO spindle_dev.index: Building SPD index with epsilon=5.552291233651296 [2026-01-19 22:34:28,487] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-19 22:34:28,503] INFO spindle_dev.index: Cluster 0: 229 SPDs, 28 blocks [2026-01-19 22:34:28,504] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-19 22:34:28,527] INFO spindle_dev.index: Finished block 0 in 0.02 seconds, found 9 clusters. [2026-01-19 22:34:28,527] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-19 22:34:28,560] INFO spindle_dev.index: Finished block 1 in 0.03 seconds, found 15 clusters. [2026-01-19 22:34:28,561] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-19 22:34:28,597] INFO spindle_dev.index: Finished block 2 in 0.04 seconds, found 17 clusters. [2026-01-19 22:34:28,598] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-19 22:34:28,631] INFO spindle_dev.index: Finished block 3 in 0.03 seconds, found 15 clusters. [2026-01-19 22:34:28,632] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-19 22:34:28,662] INFO spindle_dev.index: Finished block 4 in 0.03 seconds, found 13 clusters. [2026-01-19 22:34:28,663] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-19 22:34:28,694] INFO spindle_dev.index: Finished block 5 in 0.03 seconds, found 14 clusters. [2026-01-19 22:34:28,695] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-19 22:34:28,727] INFO spindle_dev.index: Finished block 6 in 0.03 seconds, found 14 clusters. [2026-01-19 22:34:28,727] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-19 22:34:28,766] INFO spindle_dev.index: Finished block 7 in 0.04 seconds, found 18 clusters. [2026-01-19 22:34:28,766] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-19 22:34:28,793] INFO spindle_dev.index: Finished block 8 in 0.03 seconds, found 11 clusters. [2026-01-19 22:34:28,793] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-19 22:34:28,834] INFO spindle_dev.index: Finished block 9 in 0.04 seconds, found 19 clusters. [2026-01-19 22:34:28,834] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-19 22:34:28,867] INFO spindle_dev.index: Finished block 10 in 0.03 seconds, found 13 clusters. [2026-01-19 22:34:28,868] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-19 22:34:28,886] INFO spindle_dev.index: Finished block 11 in 0.02 seconds, found 6 clusters. [2026-01-19 22:34:28,886] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-19 22:34:28,910] INFO spindle_dev.index: Finished block 12 in 0.02 seconds, found 9 clusters. [2026-01-19 22:34:28,910] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-19 22:34:28,930] INFO spindle_dev.index: Finished block 13 in 0.02 seconds, found 7 clusters. [2026-01-19 22:34:28,931] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-19 22:34:28,944] INFO spindle_dev.index: Finished block 14 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:28,944] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-19 22:34:28,962] INFO spindle_dev.index: Finished block 15 in 0.02 seconds, found 2 clusters. [2026-01-19 22:34:28,962] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-19 22:34:28,980] INFO spindle_dev.index: Finished block 16 in 0.02 seconds, found 4 clusters. [2026-01-19 22:34:28,980] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-19 22:34:29,017] INFO spindle_dev.index: Finished block 17 in 0.04 seconds, found 6 clusters. [2026-01-19 22:34:29,018] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-19 22:34:29,042] INFO spindle_dev.index: Finished block 18 in 0.02 seconds, found 5 clusters. [2026-01-19 22:34:29,043] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-19 22:34:29,066] INFO spindle_dev.index: Finished block 19 in 0.02 seconds, found 9 clusters. [2026-01-19 22:34:29,067] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-19 22:34:29,084] INFO spindle_dev.index: Finished block 20 in 0.02 seconds, found 5 clusters. [2026-01-19 22:34:29,084] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-19 22:34:29,102] INFO spindle_dev.index: Finished block 21 in 0.02 seconds, found 5 clusters. [2026-01-19 22:34:29,103] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-19 22:34:29,119] INFO spindle_dev.index: Finished block 22 in 0.02 seconds, found 5 clusters. [2026-01-19 22:34:29,120] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-19 22:34:29,135] INFO spindle_dev.index: Finished block 23 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:29,135] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-19 22:34:29,145] INFO spindle_dev.index: Finished block 24 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:29,145] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-19 22:34:29,155] INFO spindle_dev.index: Finished block 25 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:29,156] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-19 22:34:29,167] INFO spindle_dev.index: Finished block 26 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:29,168] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-19 22:34:29,238] INFO spindle_dev.index: Finished block 27 in 0.07 seconds, found 2 clusters. [2026-01-19 22:34:29,239] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-19 22:34:29,240] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-19 22:34:29,240] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-19 22:34:29,240] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-19 22:34:29,240] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-19 22:34:29,245] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-19 22:34:29,246] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-19 22:34:29,246] INFO spindle_dev.index: Processing cluster 1 [2026-01-19 22:34:29,246] INFO spindle_dev.index: Building SPD index with epsilon=6.528217907292994 [2026-01-19 22:34:29,247] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-19 22:34:29,263] INFO spindle_dev.index: Cluster 1: 212 SPDs, 30 blocks [2026-01-19 22:34:29,264] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-19 22:34:29,273] INFO spindle_dev.index: Finished block 0 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:29,274] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-19 22:34:29,285] INFO spindle_dev.index: Finished block 1 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:29,285] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-19 22:34:29,303] INFO spindle_dev.index: Finished block 2 in 0.02 seconds, found 7 clusters. [2026-01-19 22:34:29,303] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-19 22:34:29,324] INFO spindle_dev.index: Finished block 3 in 0.02 seconds, found 9 clusters. [2026-01-19 22:34:29,325] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-19 22:34:29,347] INFO spindle_dev.index: Finished block 4 in 0.02 seconds, found 10 clusters. [2026-01-19 22:34:29,348] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-19 22:34:29,370] INFO spindle_dev.index: Finished block 5 in 0.02 seconds, found 9 clusters. [2026-01-19 22:34:29,370] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-19 22:34:29,390] INFO spindle_dev.index: Finished block 6 in 0.02 seconds, found 8 clusters. [2026-01-19 22:34:29,390] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-19 22:34:29,416] INFO spindle_dev.index: Finished block 7 in 0.03 seconds, found 12 clusters. [2026-01-19 22:34:29,417] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-19 22:34:29,441] INFO spindle_dev.index: Finished block 8 in 0.02 seconds, found 11 clusters. [2026-01-19 22:34:29,442] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-19 22:34:29,465] INFO spindle_dev.index: Finished block 9 in 0.02 seconds, found 10 clusters. [2026-01-19 22:34:29,465] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-19 22:34:29,489] INFO spindle_dev.index: Finished block 10 in 0.02 seconds, found 10 clusters. [2026-01-19 22:34:29,489] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-19 22:34:29,510] INFO spindle_dev.index: Finished block 11 in 0.02 seconds, found 8 clusters. [2026-01-19 22:34:29,510] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-19 22:34:29,530] INFO spindle_dev.index: Finished block 12 in 0.02 seconds, found 7 clusters. [2026-01-19 22:34:29,530] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-19 22:34:29,550] INFO spindle_dev.index: Finished block 13 in 0.02 seconds, found 6 clusters. [2026-01-19 22:34:29,551] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-19 22:34:29,563] INFO spindle_dev.index: Finished block 14 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:29,564] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-19 22:34:29,576] INFO spindle_dev.index: Finished block 15 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:29,577] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-19 22:34:29,586] INFO spindle_dev.index: Finished block 16 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:29,587] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-19 22:34:29,597] INFO spindle_dev.index: Finished block 17 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:29,598] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-19 22:34:29,612] INFO spindle_dev.index: Finished block 18 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:29,612] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-19 22:34:29,625] INFO spindle_dev.index: Finished block 19 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:29,625] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-19 22:34:29,636] INFO spindle_dev.index: Finished block 20 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:29,636] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-19 22:34:29,652] INFO spindle_dev.index: Finished block 21 in 0.02 seconds, found 1 clusters. [2026-01-19 22:34:29,653] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-19 22:34:29,665] INFO spindle_dev.index: Finished block 22 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:29,666] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-19 22:34:29,677] INFO spindle_dev.index: Finished block 23 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:29,677] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-19 22:34:29,688] INFO spindle_dev.index: Finished block 24 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:29,689] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-19 22:34:29,698] INFO spindle_dev.index: Finished block 25 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:29,699] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-19 22:34:29,711] INFO spindle_dev.index: Finished block 26 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:29,711] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-19 22:34:29,780] INFO spindle_dev.index: Finished block 27 in 0.07 seconds, found 7 clusters. [2026-01-19 22:34:29,781] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-19 22:34:29,799] INFO spindle_dev.index: Finished block 28 in 0.02 seconds, found 6 clusters. [2026-01-19 22:34:29,799] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-19 22:34:29,817] INFO spindle_dev.index: Finished block 29 in 0.02 seconds, found 3 clusters. [2026-01-19 22:34:29,818] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-19 22:34:29,818] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-19 22:34:29,819] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-19 22:34:29,819] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-19 22:34:29,819] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-19 22:34:29,824] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-19 22:34:29,824] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-19 22:34:29,824] INFO spindle_dev.index: Processing cluster 2 [2026-01-19 22:34:29,825] INFO spindle_dev.index: Building SPD index with epsilon=6.086265230593296 [2026-01-19 22:34:29,825] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-19 22:34:29,838] INFO spindle_dev.index: Cluster 2: 177 SPDs, 26 blocks [2026-01-19 22:34:29,839] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-19 22:34:29,850] INFO spindle_dev.index: Finished block 0 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:29,850] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-19 22:34:29,862] INFO spindle_dev.index: Finished block 1 in 0.01 seconds, found 5 clusters. [2026-01-19 22:34:29,863] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-19 22:34:29,883] INFO spindle_dev.index: Finished block 2 in 0.02 seconds, found 11 clusters. [2026-01-19 22:34:29,884] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-19 22:34:29,906] INFO spindle_dev.index: Finished block 3 in 0.02 seconds, found 12 clusters. [2026-01-19 22:34:29,907] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-19 22:34:29,931] INFO spindle_dev.index: Finished block 4 in 0.02 seconds, found 14 clusters. [2026-01-19 22:34:29,932] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-19 22:34:29,963] INFO spindle_dev.index: Finished block 5 in 0.03 seconds, found 18 clusters. [2026-01-19 22:34:29,963] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-19 22:34:29,992] INFO spindle_dev.index: Finished block 6 in 0.03 seconds, found 17 clusters. [2026-01-19 22:34:29,992] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-19 22:34:30,018] INFO spindle_dev.index: Finished block 7 in 0.03 seconds, found 15 clusters. [2026-01-19 22:34:30,019] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-19 22:34:30,041] INFO spindle_dev.index: Finished block 8 in 0.02 seconds, found 12 clusters. [2026-01-19 22:34:30,041] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-19 22:34:30,066] INFO spindle_dev.index: Finished block 9 in 0.02 seconds, found 14 clusters. [2026-01-19 22:34:30,066] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-19 22:34:30,093] INFO spindle_dev.index: Finished block 10 in 0.03 seconds, found 15 clusters. [2026-01-19 22:34:30,093] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-19 22:34:30,112] INFO spindle_dev.index: Finished block 11 in 0.02 seconds, found 10 clusters. [2026-01-19 22:34:30,113] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-19 22:34:30,131] INFO spindle_dev.index: Finished block 12 in 0.02 seconds, found 9 clusters. [2026-01-19 22:34:30,131] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-19 22:34:30,142] INFO spindle_dev.index: Finished block 13 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,142] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-19 22:34:30,156] INFO spindle_dev.index: Finished block 14 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,156] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-19 22:34:30,164] INFO spindle_dev.index: Finished block 15 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:30,164] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-19 22:34:30,184] INFO spindle_dev.index: Finished block 16 in 0.02 seconds, found 2 clusters. [2026-01-19 22:34:30,185] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-19 22:34:30,198] INFO spindle_dev.index: Finished block 17 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:30,199] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-19 22:34:30,210] INFO spindle_dev.index: Finished block 18 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,210] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-19 22:34:30,232] INFO spindle_dev.index: Finished block 19 in 0.02 seconds, found 2 clusters. [2026-01-19 22:34:30,233] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-19 22:34:30,243] INFO spindle_dev.index: Finished block 20 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:30,243] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-19 22:34:30,289] INFO spindle_dev.index: Finished block 21 in 0.05 seconds, found 4 clusters. [2026-01-19 22:34:30,289] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-19 22:34:30,298] INFO spindle_dev.index: Finished block 22 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:30,299] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-19 22:34:30,307] INFO spindle_dev.index: Finished block 23 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:30,307] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-19 22:34:30,320] INFO spindle_dev.index: Finished block 24 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:30,320] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-19 22:34:30,337] INFO spindle_dev.index: Finished block 25 in 0.02 seconds, found 4 clusters. [2026-01-19 22:34:30,337] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-19 22:34:30,338] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-19 22:34:30,338] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-19 22:34:30,338] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-19 22:34:30,339] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-19 22:34:30,342] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-19 22:34:30,342] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-19 22:34:30,343] INFO spindle_dev.index: Processing cluster 3 [2026-01-19 22:34:30,343] INFO spindle_dev.index: Building SPD index with epsilon=5.8950403735406995 [2026-01-19 22:34:30,343] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-19 22:34:30,352] INFO spindle_dev.index: Cluster 3: 122 SPDs, 26 blocks [2026-01-19 22:34:30,353] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-19 22:34:30,361] INFO spindle_dev.index: Finished block 0 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:30,361] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-19 22:34:30,372] INFO spindle_dev.index: Finished block 1 in 0.01 seconds, found 7 clusters. [2026-01-19 22:34:30,372] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-19 22:34:30,382] INFO spindle_dev.index: Finished block 2 in 0.01 seconds, found 7 clusters. [2026-01-19 22:34:30,382] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-19 22:34:30,396] INFO spindle_dev.index: Finished block 3 in 0.01 seconds, found 11 clusters. [2026-01-19 22:34:30,397] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-19 22:34:30,412] INFO spindle_dev.index: Finished block 4 in 0.02 seconds, found 12 clusters. [2026-01-19 22:34:30,412] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-19 22:34:30,430] INFO spindle_dev.index: Finished block 5 in 0.02 seconds, found 15 clusters. [2026-01-19 22:34:30,430] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-19 22:34:30,453] INFO spindle_dev.index: Finished block 6 in 0.02 seconds, found 19 clusters. [2026-01-19 22:34:30,454] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-19 22:34:30,477] INFO spindle_dev.index: Finished block 7 in 0.02 seconds, found 21 clusters. [2026-01-19 22:34:30,478] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-19 22:34:30,501] INFO spindle_dev.index: Finished block 8 in 0.02 seconds, found 20 clusters. [2026-01-19 22:34:30,501] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-19 22:34:30,522] INFO spindle_dev.index: Finished block 9 in 0.02 seconds, found 18 clusters. [2026-01-19 22:34:30,523] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-19 22:34:30,554] INFO spindle_dev.index: Finished block 10 in 0.03 seconds, found 25 clusters. [2026-01-19 22:34:30,554] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-19 22:34:30,572] INFO spindle_dev.index: Finished block 11 in 0.02 seconds, found 15 clusters. [2026-01-19 22:34:30,573] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-19 22:34:30,583] INFO spindle_dev.index: Finished block 12 in 0.01 seconds, found 7 clusters. [2026-01-19 22:34:30,584] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-19 22:34:30,606] INFO spindle_dev.index: Finished block 13 in 0.02 seconds, found 16 clusters. [2026-01-19 22:34:30,606] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-19 22:34:30,618] INFO spindle_dev.index: Finished block 14 in 0.01 seconds, found 8 clusters. [2026-01-19 22:34:30,618] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-19 22:34:30,627] INFO spindle_dev.index: Finished block 15 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:30,627] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-19 22:34:30,635] INFO spindle_dev.index: Finished block 16 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:30,636] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-19 22:34:30,642] INFO spindle_dev.index: Finished block 17 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,642] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-19 22:34:30,649] INFO spindle_dev.index: Finished block 18 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,649] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-19 22:34:30,662] INFO spindle_dev.index: Finished block 19 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:30,662] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-19 22:34:30,670] INFO spindle_dev.index: Finished block 20 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,670] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-19 22:34:30,681] INFO spindle_dev.index: Finished block 21 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,681] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-19 22:34:30,695] INFO spindle_dev.index: Finished block 22 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,695] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-19 22:34:30,716] INFO spindle_dev.index: Finished block 23 in 0.02 seconds, found 2 clusters. [2026-01-19 22:34:30,717] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-19 22:34:30,725] INFO spindle_dev.index: Finished block 24 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,725] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-19 22:34:30,757] INFO spindle_dev.index: Finished block 25 in 0.03 seconds, found 5 clusters. [2026-01-19 22:34:30,757] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-19 22:34:30,758] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-19 22:34:30,758] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-19 22:34:30,758] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-19 22:34:30,759] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-19 22:34:30,762] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-19 22:34:30,762] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-19 22:34:30,763] INFO spindle_dev.index: Processing cluster 4 [2026-01-19 22:34:30,763] INFO spindle_dev.index: Building SPD index with epsilon=8.06822025162069 [2026-01-19 22:34:30,764] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-19 22:34:30,772] INFO spindle_dev.index: Cluster 4: 106 SPDs, 24 blocks [2026-01-19 22:34:30,772] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-19 22:34:30,777] INFO spindle_dev.index: Finished block 0 in 0.00 seconds, found 1 clusters. [2026-01-19 22:34:30,778] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-19 22:34:30,783] INFO spindle_dev.index: Finished block 1 in 0.00 seconds, found 2 clusters. [2026-01-19 22:34:30,783] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-19 22:34:30,788] INFO spindle_dev.index: Finished block 2 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,788] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-19 22:34:30,795] INFO spindle_dev.index: Finished block 3 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,795] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-19 22:34:30,801] INFO spindle_dev.index: Finished block 4 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,801] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-19 22:34:30,808] INFO spindle_dev.index: Finished block 5 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,808] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-19 22:34:30,813] INFO spindle_dev.index: Finished block 6 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,813] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-19 22:34:30,820] INFO spindle_dev.index: Finished block 7 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,820] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-19 22:34:30,826] INFO spindle_dev.index: Finished block 8 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,826] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-19 22:34:30,832] INFO spindle_dev.index: Finished block 9 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,833] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-19 22:34:30,840] INFO spindle_dev.index: Finished block 10 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,840] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-19 22:34:30,846] INFO spindle_dev.index: Finished block 11 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:30,846] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-19 22:34:30,853] INFO spindle_dev.index: Finished block 12 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,853] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-19 22:34:30,881] INFO spindle_dev.index: Finished block 13 in 0.03 seconds, found 4 clusters. [2026-01-19 22:34:30,882] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-19 22:34:30,889] INFO spindle_dev.index: Finished block 14 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,889] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-19 22:34:30,926] INFO spindle_dev.index: Finished block 15 in 0.04 seconds, found 11 clusters. [2026-01-19 22:34:30,926] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-19 22:34:30,933] INFO spindle_dev.index: Finished block 16 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,934] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-19 22:34:30,939] INFO spindle_dev.index: Finished block 17 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,940] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-19 22:34:30,949] INFO spindle_dev.index: Finished block 18 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,949] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-19 22:34:30,956] INFO spindle_dev.index: Finished block 19 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,957] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-19 22:34:30,963] INFO spindle_dev.index: Finished block 20 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,963] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-19 22:34:30,972] INFO spindle_dev.index: Finished block 21 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,972] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-19 22:34:30,981] INFO spindle_dev.index: Finished block 22 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:30,981] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-19 22:34:30,990] INFO spindle_dev.index: Finished block 23 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:30,990] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-19 22:34:30,990] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-19 22:34:30,991] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-19 22:34:30,991] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-19 22:34:30,991] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-19 22:34:30,993] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-19 22:34:30,993] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-19 22:34:30,994] INFO spindle_dev.index: Processing cluster 5 [2026-01-19 22:34:30,994] INFO spindle_dev.index: Building SPD index with epsilon=7.23115036152569 [2026-01-19 22:34:30,994] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-19 22:34:31,002] INFO spindle_dev.index: Cluster 5: 101 SPDs, 28 blocks [2026-01-19 22:34:31,003] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-19 22:34:31,007] INFO spindle_dev.index: Finished block 0 in 0.00 seconds, found 1 clusters. [2026-01-19 22:34:31,007] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-19 22:34:31,012] INFO spindle_dev.index: Finished block 1 in 0.00 seconds, found 2 clusters. [2026-01-19 22:34:31,012] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-19 22:34:31,018] INFO spindle_dev.index: Finished block 2 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:31,019] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-19 22:34:31,025] INFO spindle_dev.index: Finished block 3 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:31,026] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-19 22:34:31,033] INFO spindle_dev.index: Finished block 4 in 0.01 seconds, found 5 clusters. [2026-01-19 22:34:31,033] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-19 22:34:31,041] INFO spindle_dev.index: Finished block 5 in 0.01 seconds, found 5 clusters. [2026-01-19 22:34:31,041] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-19 22:34:31,049] INFO spindle_dev.index: Finished block 6 in 0.01 seconds, found 6 clusters. [2026-01-19 22:34:31,049] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-19 22:34:31,062] INFO spindle_dev.index: Finished block 7 in 0.01 seconds, found 5 clusters. [2026-01-19 22:34:31,062] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-19 22:34:31,070] INFO spindle_dev.index: Finished block 8 in 0.01 seconds, found 5 clusters. [2026-01-19 22:34:31,070] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-19 22:34:31,077] INFO spindle_dev.index: Finished block 9 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:31,077] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-19 22:34:31,083] INFO spindle_dev.index: Finished block 10 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:31,083] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-19 22:34:31,091] INFO spindle_dev.index: Finished block 11 in 0.01 seconds, found 5 clusters. [2026-01-19 22:34:31,091] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-19 22:34:31,097] INFO spindle_dev.index: Finished block 12 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:31,098] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-19 22:34:31,104] INFO spindle_dev.index: Finished block 13 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:31,105] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-19 22:34:31,116] INFO spindle_dev.index: Finished block 14 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:31,116] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-19 22:34:31,123] INFO spindle_dev.index: Finished block 15 in 0.01 seconds, found 5 clusters. [2026-01-19 22:34:31,124] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-19 22:34:31,129] INFO spindle_dev.index: Finished block 16 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:31,129] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-19 22:34:31,136] INFO spindle_dev.index: Finished block 17 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:31,136] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-19 22:34:31,145] INFO spindle_dev.index: Finished block 18 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:31,145] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-19 22:34:31,151] INFO spindle_dev.index: Finished block 19 in 0.01 seconds, found 1 clusters. [2026-01-19 22:34:31,151] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-19 22:34:31,159] INFO spindle_dev.index: Finished block 20 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:31,159] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-19 22:34:31,166] INFO spindle_dev.index: Finished block 21 in 0.01 seconds, found 4 clusters. [2026-01-19 22:34:31,166] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-19 22:34:31,175] INFO spindle_dev.index: Finished block 22 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:31,176] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-19 22:34:31,183] INFO spindle_dev.index: Finished block 23 in 0.01 seconds, found 2 clusters. [2026-01-19 22:34:31,183] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-19 22:34:31,191] INFO spindle_dev.index: Finished block 24 in 0.01 seconds, found 3 clusters. [2026-01-19 22:34:31,191] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-19 22:34:31,196] INFO spindle_dev.index: Finished block 25 in 0.00 seconds, found 1 clusters. [2026-01-19 22:34:31,196] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-19 22:34:31,200] INFO spindle_dev.index: Finished block 26 in 0.00 seconds, found 1 clusters. [2026-01-19 22:34:31,201] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-19 22:34:31,235] INFO spindle_dev.index: Finished block 27 in 0.03 seconds, found 10 clusters. [2026-01-19 22:34:31,236] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-19 22:34:31,236] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-19 22:34:31,236] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-19 22:34:31,237] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-19 22:34:31,237] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-19 22:34:31,240] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-19 22:34:31,240] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances.
In [19]:
search_cfg = search.SearchConfig(max_results=2, debug=False, max_failed_starts=5, max_failed_paths=10)
In [20]:
test_results = test.run_sanity_search(data, dag_dict, config, search_cfg, max_queries=2000, skip_baseline=True)
[2026-01-19 22:34:43,473] INFO spindle_dev.test: Built ground-truth paths for 229 SPDs in cluster 0 across 28 blocks. [2026-01-19 22:34:43,530] INFO spindle_dev.test: Built ground-truth paths for 212 SPDs in cluster 1 across 30 blocks. [2026-01-19 22:34:43,567] INFO spindle_dev.test: Built ground-truth paths for 177 SPDs in cluster 2 across 26 blocks. [2026-01-19 22:34:43,588] INFO spindle_dev.test: Built ground-truth paths for 122 SPDs in cluster 3 across 26 blocks. [2026-01-19 22:34:43,599] INFO spindle_dev.test: Built ground-truth paths for 106 SPDs in cluster 4 across 24 blocks. [2026-01-19 22:34:43,613] INFO spindle_dev.test: Built ground-truth paths for 101 SPDs in cluster 5 across 28 blocks. [2026-01-19 22:34:43,613] INFO spindle_dev.test: Created ground-truth paths for all clusters. [2026-01-19 22:34:43,613] INFO spindle_dev.test: Search config: SearchConfig(max_results=2, max_failed_starts=5, max_failed_paths=10, total_paths_limit=1000, deterministic=DeterministicConfig(seed=0), debug=False) [2026-01-19 22:34:43,615] INFO spindle_dev.test: Running sanity search with 947 queries. 100%|██████████| 947/947 [00:05<00:00, 184.33it/s] [2026-01-19 22:34:48,755] INFO spindle_dev.test: Sanity search: 947 queries, 947 exact path matches, 947 leaf matches, mean search_time=0.0048s
In [22]:
fig = plotting.visualize_block_dag_sankey_scaled(dag_dict[0].nodes, height=800, thickness=20.0)
fig
In [23]:
import pandas as pd
def get_dag_stats(dag_dict):
dag_stats = []
for cluster_id, cluster_dag in dag_dict.items():
block_to_node = cluster_dag.block_to_node_indices
for node in cluster_dag.nodes:
num_spds = len(node.metadata.members)
block_size = node.metadata.mean.shape[0]
block_id = node.block_index
radius = node.metadata.radius
#print(f"Cluster_id {cluster_id} Node id {node.global_node_id} Block {block_id}: {num_spds} spds")
dag_stats.append({
"cluster_id": cluster_id,
"node_id": node.global_node_id,
"block_id": block_id,
"block_cluster_id": node.block_cluster_id,
"num_spds": num_spds,
"radius": radius,
"block_size": block_size
})
dag_stats_df = pd.DataFrame(dag_stats)
return dag_stats_df
In [24]:
dag_stats_df = get_dag_stats(dag_dict)
In [25]:
score_list = []
for cluster_id in set(data.labels):
cluster_dag = dag_dict[cluster_id]
cluster_score = index.score_nodes_from_a_cluster(data, cluster_id, dag_dict, take_representative_mean=True)
score_list.extend(cluster_score)
100%|██████████| 231/231 [00:00<00:00, 679.83it/s] 100%|██████████| 155/155 [00:00<00:00, 509.34it/s] 100%|██████████| 183/183 [00:00<00:00, 631.97it/s] 100%|██████████| 238/238 [00:00<00:00, 549.45it/s] 100%|██████████| 66/66 [00:00<00:00, 187.38it/s] 100%|██████████| 97/97 [00:00<00:00, 152.40it/s]
In [26]:
node_scores = []
for node_score in score_list:
node_scores.append({
"cluster_id": node_score["cluster_id"],
"block_id": node_score["block_id"],
"node_id": node_score["node_id"],
"node_score": node_score["node_score"],
"num_spds": node_score["num_spds"],
"block_size": node_score["block_size"],
"E": node_score["E"],
"Q": node_score["Q"],
"S": node_score["S"],
"num_modules": len(node_score["modules"])
})
node_score_df = pd.DataFrame(node_scores)
In [27]:
node_score_df.sort_values(by='node_score', ascending=False).head(20)
Out[27]:
| cluster_id | block_id | node_id | node_score | num_spds | block_size | E | Q | S | num_modules | |
|---|---|---|---|---|---|---|---|---|---|---|
| 846 | 4 | 15 | 39 | 0.264200 | 1 | 47 | 3.462450 | 0.177001 | 0.568906 | 2 |
| 852 | 4 | 15 | 45 | 0.236515 | 1 | 47 | 3.026848 | 0.180715 | 0.567616 | 2 |
| 558 | 2 | 21 | 172 | 0.219011 | 174 | 44 | 2.068550 | 0.254281 | 0.583624 | 3 |
| 559 | 2 | 21 | 173 | 0.216651 | 1 | 44 | 1.605486 | 0.217354 | 0.379153 | 6 |
| 848 | 4 | 15 | 41 | 0.213373 | 1 | 47 | 2.803306 | 0.158580 | 0.520024 | 3 |
| 961 | 5 | 27 | 88 | 0.183719 | 1 | 47 | 2.286959 | 0.135007 | 0.404971 | 3 |
| 928 | 5 | 14 | 55 | 0.179217 | 2 | 22 | 0.953855 | 0.357961 | 0.475121 | 4 |
| 705 | 3 | 10 | 136 | 0.178715 | 2 | 18 | 0.630332 | 0.444707 | 0.362446 | 7 |
| 963 | 5 | 27 | 90 | 0.174978 | 1 | 47 | 2.729755 | 0.135573 | 0.527192 | 2 |
| 844 | 4 | 15 | 37 | 0.174616 | 1 | 47 | 3.286446 | 0.114943 | 0.537754 | 2 |
| 370 | 1 | 27 | 139 | 0.157943 | 206 | 49 | 1.902618 | 0.159266 | 0.478776 | 4 |
| 805 | 3 | 25 | 236 | 0.154702 | 1 | 44 | 1.311143 | 0.187637 | 0.371180 | 6 |
| 755 | 3 | 13 | 186 | 0.151838 | 1 | 17 | 0.940500 | 0.406067 | 0.602421 | 4 |
| 965 | 5 | 27 | 92 | 0.150483 | 1 | 47 | 2.733197 | 0.126052 | 0.563219 | 5 |
| 845 | 4 | 15 | 38 | 0.146679 | 1 | 47 | 3.540809 | 0.116305 | 0.643825 | 2 |
| 385 | 1 | 29 | 154 | 0.142854 | 9 | 20 | 0.732760 | 0.307037 | 0.365051 | 4 |
| 839 | 4 | 13 | 32 | 0.141916 | 1 | 45 | 2.179629 | 0.123284 | 0.471873 | 4 |
| 966 | 5 | 27 | 93 | 0.141826 | 1 | 47 | 3.097986 | 0.119917 | 0.618238 | 3 |
| 929 | 5 | 14 | 56 | 0.139772 | 1 | 22 | 1.148377 | 0.243380 | 0.499909 | 5 |
| 804 | 3 | 25 | 235 | 0.138025 | 1 | 44 | 2.495430 | 0.124211 | 0.554703 | 4 |
In [42]:
id = 852
scores_846, fig_846 = plotting.plot_module_heatmap_plus_spatial(
id,
score_list,
node_score_df,
dag_stats_df,
dag_dict,
data,
adata
)
Creating combined figure for cluster 4, node 45, block 15
In [43]:
fig_846.savefig(f"{result_dir}/{id}_module_0_1_enrichment.png", dpi=300)
In [44]:
from spindle_dev import go_score
In [47]:
out_846 = go_score.enrich_modules_with_gseapy(score_list[id]['modules'])
Module 0: enrichment completed with 473 terms Module 1: enrichment completed with 503 terms
In [54]:
importlib.reload(go_score)
Out[54]:
<module 'spindle_dev.go_score' from '/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/go_score.py'>
In [59]:
for module_id, module_genes, res in out_846:
if module_id == 0:
print(module_genes)
fig, axes = go_score.plot_module_enrichment_libraries_2(
module_id=module_id,
module_genes=module_genes,
res=res,
top_n=7,
ncols=2,
bar_color="#D41515EE", # pick your color
tick_fontsize=10,
label_fontsize=10,
title_fontsize=8,
compact_height_per_term=0.2,
min_fig_h=8,
plot_kind="dot",
size_range=(20, 200),
use_constrained_layout=False,
save_dir=f"{result_dir}/go_enrichment/enrichment_{id}"
)
if fig is not None:
plt.show()
['AHNAK2', 'CAPNS2', 'CLDN1', 'CRABP2', 'DSC1', 'DSP', 'ENAH', 'GPR157', 'KDM5B', 'KRT16', 'KRT17', 'KRT2', 'KRTDAP', 'LY6D', 'LYPD3', 'MAL2', 'NOTCH3', 'PERP', 'RHOV', 'S100A14', 'SERPINB5', 'SLPI', 'TUBB2B', 'VEGFA']
/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/go_score.py:519: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
In [48]:
for module_id, module_genes, res in out_846:
if module_id == 1:
print(module_genes)
fig, axes = go_score.plot_module_enrichment_libraries(
module_id=module_id,
module_genes=module_genes,
res=res,
top_n=7,
ncols=2,
bar_color="#D41515EE", # pick your color
tick_fontsize=10,
label_fontsize=10,
title_fontsize=8,
compact_height_per_term=0.2,
min_fig_h=6,
save_dir=f"{result_dir}/go_enrichment/enrichment_{id}"
)
if fig is not None:
plt.show()
['ACER1', 'AQP3', 'AREG', 'ARG1', 'AZGP1', 'C15orf48', 'CLDN4', 'CXADR', 'DEFB1', 'DMKN', 'ID4', 'IFI27', 'IFI6', 'KRT1', 'LOR', 'PTEN', 'PTGER3', 'PTGR1', 'SBSN', 'SERPINB2', 'SLC25A39', 'TACSTD2', 'TMEM45A'] ['GO_Biological_Process_2023', 'KEGG_2021_Human', 'MSigDB_Hallmark_2020', 'Reactome_2022']
In [37]:
# Create a dictionary of tile_id to block_id
# Clusters do not share tile ids across them
from collections import defaultdict
tile_to_block_dict = {}
cluster_to_tile = defaultdict(set)
for cluster_id, cluster_dag in dag_dict.items():
block_to_node = cluster_dag.block_to_node_indices
for node in cluster_dag.nodes:
member_tile_ids = [tid for tid,_ in node.metadata.members]
for tile_id in member_tile_ids:
if not tile_id in tile_to_block_dict:
tile_to_block_dict[tile_id] = {}
tile_to_block_dict[tile_id][node.block_index] = node.block_cluster_id
cluster_to_tile[cluster_id].add(tile_id)
In [38]:
def get_spot_score_dict(block_id, cluster_id, tile_to_block_dict, grid=False):
spot_score = {}
tiles_of_cluster = cluster_to_tile[cluster_id]
for tid in tiles_of_cluster:
if tid not in tile_to_block_dict:
continue
c = tile_to_block_dict[tid][block_id]
t = data.metadata['tiles'][tid]
for id in t.idx:
spot_score[id] = c
fig, ax = plotting.plot_spot_module_scores(spot_score, tiles, adata, color='discrete', grid=grid)
return fig, ax
In [41]:
fig, ax = get_spot_score_dict(5, 4, tile_to_block_dict, grid=False)
In [ ]:
sc.pl.spatial(adata,color=['AQP3', 'AHNAK2'], spot_size=20, cmap='Reds')
/tmp/ipykernel_3999659/909464612.py:1: FutureWarning: Use `squidpy.pl.spatial_scatter` instead.
In [ ]: