In [1]:
# Set up sys.path so that 'src/spindle_dev' is importable as 'spindle_dev'
import sys
import importlib
from pathlib import Path
project_root = '/data/sarkar_lab/Projects/spindle_dev'
src_path = Path(project_root) / 'src'
if str(src_path) not in sys.path:
sys.path.insert(0, str(src_path))
import spindle_dev
# Reload to pick up code changes without restarting the kernel
importlib.reload(spindle_dev)
Out[1]:
<module 'spindle_dev' from '/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/__init__.py'>
In [2]:
import scanpy as sc
In [3]:
import glob
In [4]:
h5ad_files = glob.glob("/data/sarkar_lab/insitupy_demo_data_xenium/*.h5ad")
In [5]:
h5ad_files
Out[5]:
['/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_skin_melanoma.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_brain_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_kidney_nondiseased.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lung_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lymph_node.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lymph_node_5k.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_pancreatic_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_breast_cancer.h5ad']
In [6]:
adata = sc.read_h5ad('/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lung_cancer.h5ad')
In [15]:
kept_indices = adata.obsm['spatial'][:,1] < 3400
adata = adata[kept_indices].copy()
In [16]:
from pathlib import Path
import spindle_dev
import spindle_dev.metrics as metrics
import spindle_dev.index as index
import spindle_dev.preprocessing as preprocessing
import spindle_dev.plotting as plotting
import spindle_dev.test as test
import spindle_dev.search as search
import spindle_dev.typing as typing
# Reload to pick up code changes without restarting the kernel
import time
import scanpy as sc
import glob
import pandas as pd
import numpy as np
from concurrent.futures import ThreadPoolExecutor, as_completed
from joblib import Parallel, delayed
import argparse
In [17]:
result_dir = "/data/sarkar_lab/Projects/spindle_dev/results/hlung_10X/"
Path(result_dir).mkdir(parents=True, exist_ok=True)
In [18]:
all_genes = True
resolution=0.5
min_final_size=10
start_time = time.time()
coords = adata.obsm["spatial"]
tiles = preprocessing.build_quadtree_tiles(coords, max_pts=200, min_side=0.0, max_depth=40)
# remove tiles with less than 5 spots
tiles = [tile for tile in tiles if len(tile.idx) >= 5]
tiles = preprocessing.reindex_tiles(tiles)
num_genes = adata.n_vars
genes_work, gene_idx = spindle_dev.preprocessing.topvar_genes(adata, G=num_genes)
tile_covs = spindle_dev.preprocessing.build_tile_covs_full_serial(adata, tiles, gene_idx, eps=1e-6)
data = index.ProcessedData(tiles, tile_covs, genes_work, adata.n_obs)
if 'pca' not in data.latent:
data.reduce_dim(num_pca_components=30, n_components=2, do_umap=True)
data.cluster_spds(cluster_distance="tree", cluster_method="leiden", resolution=resolution)
data.assign_label_to_spots()
data.get_corr_mean_by_cluster()
out_dict = data.get_adaptive_runs(find_blocks=True, with_size_guard=True,min_final_size=min_final_size,max_final_size=100)
epsilon_block_wise_dict = {}
epsilon_dict = {}
for cluster_id in set(data.labels):
eps_per_block, eps_elbow_per_block, eps = index.choose_adaptive_epsilons(data, cluster_id, k_target_per_block=64)
epsilon_block_wise_dict[int(cluster_id)] = eps_elbow_per_block
epsilon_dict[int(cluster_id)] = eps
[2026-01-20 07:00:49,113] INFO spindle_dev.index: Clustering SPD-s using 'tree' distance. [2026-01-20 07:00:49,114] INFO spindle_dev.index: Building ultrametric features from SPD matrices. /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/scipy/cluster/hierarchy.py:1673: RuntimeWarning: invalid value encountered in scalar divide c = xp.sum(numerator) / xp.sqrt(xp.sum(denomA) * xp.sum(denomB)) [2026-01-20 07:00:55,467] INFO spindle_dev.index: Computing latent features from the tree representations. [2026-01-20 07:01:01,953] INFO spindle_dev.index: Reducing latent features to 30 dimensions using PCA. [2026-01-20 07:01:17,811] INFO spindle_dev.index: Explained variance ratios by PCA components: [0.02745872 0.02069802 0.01133705 0.00752258 0.00612175 0.00553195 0.00462097 0.00438043 0.00391815 0.00342668 0.00339359 0.00313932 0.00301993 0.00281963 0.00278662 0.00274886 0.00263091 0.00254829 0.00252638 0.00248274 0.00246482 0.00243376 0.00236487 0.00235165 0.00231674 0.00228558 0.00227759 0.0022568 0.00223588 0.00220875] [2026-01-20 07:01:17,812] INFO spindle_dev.index: Reducing latent features to 2 dimensions using UMAP. /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/umap/umap_.py:1952: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism. warn( [2026-01-20 07:01:21,369] INFO spindle_dev.index: Clustering SPD-s using 'tree' distance. [2026-01-20 07:01:21,370] INFO spindle_dev.index: Clustering SPD matrices using Leiden clustering with resolution 0.50. [2026-01-20 07:01:21,546] INFO spindle_dev.index: Since clustering method is tree, I am going to find global order per cluster [2026-01-20 07:01:21,547] INFO spindle_dev.index: Finding consensus tree for cluster 0 [2026-01-20 07:01:26,729] INFO spindle_dev.index: Finding consensus tree for cluster 1 [2026-01-20 07:01:26,912] INFO spindle_dev.index: Finding consensus tree for cluster 2 [2026-01-20 07:01:27,010] INFO spindle_dev.index: Finding consensus tree for cluster 3 [2026-01-20 07:01:27,041] INFO spindle_dev.index: Finding consensus tree for cluster 4 [2026-01-20 07:01:27,063] INFO spindle_dev.index: Finding consensus tree for cluster 5 [2026-01-20 07:01:32,758] INFO spindle_dev.index: Computing mean correlation matrix for cluster 0 [2026-01-20 07:01:36,510] INFO spindle_dev.index: Computing mean correlation matrix for cluster 1 [2026-01-20 07:01:39,104] INFO spindle_dev.index: Computing mean correlation matrix for cluster 2 [2026-01-20 07:01:39,418] INFO spindle_dev.index: Computing mean correlation matrix for cluster 3 [2026-01-20 07:01:39,619] INFO spindle_dev.index: Computing mean correlation matrix for cluster 4 [2026-01-20 07:01:39,668] INFO spindle_dev.index: Computing mean correlation matrix for cluster 5 [2026-01-20 07:01:39,697] INFO spindle_dev.index: Finding adaptive block runs for cluster 0 [2026-01-20 07:01:40,120] INFO spindle_dev.index: Chose t=0.8691020228638607 resulting in 243 blocks instead of 142 blocks would have gotten by default [2026-01-20 07:01:40,166] INFO spindle_dev.index: Final block runs for cluster 0: 28 blocks. [2026-01-20 07:01:40,166] INFO spindle_dev.index: Finding adaptive block runs for cluster 1 [2026-01-20 07:01:40,552] INFO spindle_dev.index: Chose t=0.8916213994312294 resulting in 239 blocks instead of 163 blocks would have gotten by default [2026-01-20 07:01:40,595] INFO spindle_dev.index: Final block runs for cluster 1: 27 blocks. [2026-01-20 07:01:40,596] INFO spindle_dev.index: Finding adaptive block runs for cluster 2 [2026-01-20 07:01:40,979] INFO spindle_dev.index: Chose t=0.8980608363535082 resulting in 267 blocks instead of 195 blocks would have gotten by default [2026-01-20 07:01:41,032] INFO spindle_dev.index: Final block runs for cluster 2: 29 blocks. [2026-01-20 07:01:41,032] INFO spindle_dev.index: Finding adaptive block runs for cluster 3 [2026-01-20 07:01:41,413] INFO spindle_dev.index: Chose t=0.9206832628186625 resulting in 201 blocks instead of 201 blocks would have gotten by default [2026-01-20 07:01:41,447] INFO spindle_dev.index: Final block runs for cluster 3: 24 blocks. [2026-01-20 07:01:41,448] INFO spindle_dev.index: Finding adaptive block runs for cluster 4 [2026-01-20 07:01:41,829] INFO spindle_dev.index: Chose t=0.9186079707439211 resulting in 168 blocks instead of 166 blocks would have gotten by default [2026-01-20 07:01:41,854] INFO spindle_dev.index: Final block runs for cluster 4: 22 blocks. [2026-01-20 07:01:41,854] INFO spindle_dev.index: Finding adaptive block runs for cluster 5 [2026-01-20 07:01:42,237] INFO spindle_dev.index: Chose t=0.8442664578756953 resulting in 243 blocks instead of 115 blocks would have gotten by default [2026-01-20 07:01:42,280] INFO spindle_dev.index: Final block runs for cluster 5: 31 blocks.
In [19]:
import matplotlib.pylab as plt
from matplotlib.patches import Rectangle
from matplotlib.collections import PatchCollection
In [20]:
import numpy as np
palette = sc.pl.palettes.default_20
point_colors = [palette[lab] for lab in data.spot_label.values()]
labels = data.labels
indices = np.array([int(i) for i in data.spot_label.keys()])
coords = adata.obsm['spatial']
plt.scatter(coords[indices,0], coords[indices,1], s=1, c=point_colors, lw=0, alpha=0.8)
patches = []
for t in tiles:
bbox = t.bbox if hasattr(t, "bbox") else t["bbox"] # (xmin,ymin,xmax,ymax)
x0, y0, x1, y1 = bbox
patches.append(Rectangle((x0, y0), x1 - x0, y1 - y0, fill=False))
pc = PatchCollection(patches, match_original=True, linewidths=0.2, edgecolors='b', alpha=0.3)
# add collection
ax = plt.gca()
ax.add_collection(pc)
ax.invert_yaxis()
ax.axis('off')
#ax.set_aspect('equal')
plt.savefig(f"{result_dir}/spatial_plot_with_cluster.png", dpi=300)
plt.show()
In [ ]:
In [21]:
# Create colors for clusters
import numpy as np
import matplotlib.pyplot as plt
n_clusters = len(set(data.labels))
# Use Scanpy's default_20 palette for consistency
palette = sc.pl.palettes.default_20
cluster_colors = [palette[lab] for lab in data.labels]
plt.scatter(data.latent['umap'][:,0], data.latent['umap'][:,1], s=6, lw=0, alpha=0.8, c=cluster_colors)
# Write label on the plot
# with backgrond circle for better visibility
# and bold font
for i in range(n_clusters):
cluster_points = data.latent['umap'][data.labels == i]
if len(cluster_points) == 0:
continue
x_mean = np.mean(cluster_points[:,0])
y_mean = np.mean(cluster_points[:,1])
plt.text(x_mean, y_mean, str(i), color='black', fontsize=10, ha='center', va='center', fontweight='bold',
bbox=dict(facecolor='white', edgecolor='black', boxstyle='circle,pad=0.5', alpha=0.7))
# take away top and right border
ax = plt.gca()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
# remove axis ticks
ax.set_xticks([])
ax.set_yticks([])
plt.savefig(f"{result_dir}/umap_plot_with_cluster_labels.png", dpi=300)
plt.show()
In [22]:
epsilon_block_wise_dict = {}
epsilon_dict = {}
for cluster_id in set(data.labels):
eps_per_block, eps_elbow_per_block, eps = index.choose_adaptive_epsilons(data, cluster_id, k_target_per_block=64)
epsilon_block_wise_dict[int(cluster_id)] = eps_elbow_per_block
epsilon_dict[int(cluster_id)] = eps
In [23]:
epsilon_dict
Out[23]:
{0: np.float64(7.899105295225772),
1: np.float64(7.333615752282477),
2: np.float64(7.253261245720598),
3: np.float64(7.55950504648754),
4: np.float64(6.14807850563238),
5: np.float64(3.819779091158857)}
In [24]:
config = typing.IndexConfig()
config.epsilon_dict = epsilon_dict
config.epsilon_block_wise_dict = epsilon_block_wise_dict
config.threshold_type = 'constant'
config.kmean_method = 'epsilon_net'
In [25]:
# Create index
dag_dict, stat, dist_list = index.index_spds(data, config=config)
[2026-01-20 07:07:11,573] INFO spindle_dev.index: Processing cluster 0 [2026-01-20 07:07:11,574] INFO spindle_dev.index: Building SPD index with epsilon=7.899105295225772 [2026-01-20 07:07:11,576] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-20 07:07:11,607] INFO spindle_dev.index: Cluster 0: 872 SPDs, 28 blocks [2026-01-20 07:07:11,608] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-20 07:07:11,637] INFO spindle_dev.index: Finished block 0 in 0.03 seconds, found 1 clusters. [2026-01-20 07:07:11,638] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-20 07:07:11,674] INFO spindle_dev.index: Finished block 1 in 0.04 seconds, found 2 clusters. [2026-01-20 07:07:11,674] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-20 07:07:11,710] INFO spindle_dev.index: Finished block 2 in 0.04 seconds, found 2 clusters. [2026-01-20 07:07:11,711] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-20 07:07:11,748] INFO spindle_dev.index: Finished block 3 in 0.04 seconds, found 2 clusters. [2026-01-20 07:07:11,749] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-20 07:07:11,793] INFO spindle_dev.index: Finished block 4 in 0.04 seconds, found 3 clusters. [2026-01-20 07:07:11,794] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-20 07:07:11,834] INFO spindle_dev.index: Finished block 5 in 0.04 seconds, found 2 clusters. [2026-01-20 07:07:11,835] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-20 07:07:11,873] INFO spindle_dev.index: Finished block 6 in 0.04 seconds, found 2 clusters. [2026-01-20 07:07:11,874] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-20 07:07:11,912] INFO spindle_dev.index: Finished block 7 in 0.04 seconds, found 2 clusters. [2026-01-20 07:07:11,913] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-20 07:07:11,952] INFO spindle_dev.index: Finished block 8 in 0.04 seconds, found 2 clusters. [2026-01-20 07:07:11,952] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-20 07:07:11,991] INFO spindle_dev.index: Finished block 9 in 0.04 seconds, found 2 clusters. [2026-01-20 07:07:11,992] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-20 07:07:12,032] INFO spindle_dev.index: Finished block 10 in 0.04 seconds, found 2 clusters. [2026-01-20 07:07:12,033] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-20 07:07:12,072] INFO spindle_dev.index: Finished block 11 in 0.04 seconds, found 2 clusters. [2026-01-20 07:07:12,073] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-20 07:07:12,127] INFO spindle_dev.index: Finished block 12 in 0.05 seconds, found 4 clusters. [2026-01-20 07:07:12,128] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-20 07:07:12,175] INFO spindle_dev.index: Finished block 13 in 0.05 seconds, found 3 clusters. [2026-01-20 07:07:12,176] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-20 07:07:12,229] INFO spindle_dev.index: Finished block 14 in 0.05 seconds, found 4 clusters. [2026-01-20 07:07:12,230] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-20 07:07:12,277] INFO spindle_dev.index: Finished block 15 in 0.05 seconds, found 3 clusters. [2026-01-20 07:07:12,278] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-20 07:07:12,330] INFO spindle_dev.index: Finished block 16 in 0.05 seconds, found 3 clusters. [2026-01-20 07:07:12,330] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-20 07:07:12,384] INFO spindle_dev.index: Finished block 17 in 0.05 seconds, found 4 clusters. [2026-01-20 07:07:12,385] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-20 07:07:12,459] INFO spindle_dev.index: Finished block 18 in 0.07 seconds, found 6 clusters. [2026-01-20 07:07:12,460] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-20 07:07:12,509] INFO spindle_dev.index: Finished block 19 in 0.05 seconds, found 3 clusters. [2026-01-20 07:07:12,509] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-20 07:07:12,840] INFO spindle_dev.index: Finished block 20 in 0.33 seconds, found 7 clusters. [2026-01-20 07:07:12,841] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-20 07:07:12,896] INFO spindle_dev.index: Finished block 21 in 0.06 seconds, found 4 clusters. [2026-01-20 07:07:12,897] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-20 07:07:12,962] INFO spindle_dev.index: Finished block 22 in 0.07 seconds, found 3 clusters. [2026-01-20 07:07:12,963] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-20 07:07:13,061] INFO spindle_dev.index: Finished block 23 in 0.10 seconds, found 5 clusters. [2026-01-20 07:07:13,061] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-20 07:07:13,125] INFO spindle_dev.index: Finished block 24 in 0.06 seconds, found 4 clusters. [2026-01-20 07:07:13,126] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-20 07:07:13,205] INFO spindle_dev.index: Finished block 25 in 0.08 seconds, found 4 clusters. [2026-01-20 07:07:13,206] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-20 07:07:13,265] INFO spindle_dev.index: Finished block 26 in 0.06 seconds, found 3 clusters. [2026-01-20 07:07:13,266] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-20 07:07:13,328] INFO spindle_dev.index: Finished block 27 in 0.06 seconds, found 3 clusters. [2026-01-20 07:07:13,329] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-20 07:07:13,329] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-20 07:07:13,330] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-20 07:07:13,330] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-20 07:07:13,330] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-20 07:07:13,344] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-20 07:07:13,345] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-20 07:07:13,345] INFO spindle_dev.index: Processing cluster 1 [2026-01-20 07:07:13,345] INFO spindle_dev.index: Building SPD index with epsilon=7.333615752282477 [2026-01-20 07:07:13,345] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-20 07:07:14,926] INFO spindle_dev.index: Cluster 1: 526 SPDs, 27 blocks [2026-01-20 07:07:14,927] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-20 07:07:14,953] INFO spindle_dev.index: Finished block 0 in 0.03 seconds, found 3 clusters. [2026-01-20 07:07:14,953] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-20 07:07:14,980] INFO spindle_dev.index: Finished block 1 in 0.03 seconds, found 3 clusters. [2026-01-20 07:07:14,981] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-20 07:07:15,007] INFO spindle_dev.index: Finished block 2 in 0.03 seconds, found 3 clusters. [2026-01-20 07:07:15,008] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-20 07:07:15,039] INFO spindle_dev.index: Finished block 3 in 0.03 seconds, found 4 clusters. [2026-01-20 07:07:15,039] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-20 07:07:15,070] INFO spindle_dev.index: Finished block 4 in 0.03 seconds, found 4 clusters. [2026-01-20 07:07:15,071] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-20 07:07:15,098] INFO spindle_dev.index: Finished block 5 in 0.03 seconds, found 3 clusters. [2026-01-20 07:07:15,099] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-20 07:07:15,130] INFO spindle_dev.index: Finished block 6 in 0.03 seconds, found 4 clusters. [2026-01-20 07:07:15,131] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-20 07:07:15,170] INFO spindle_dev.index: Finished block 7 in 0.04 seconds, found 6 clusters. [2026-01-20 07:07:15,171] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-20 07:07:15,212] INFO spindle_dev.index: Finished block 8 in 0.04 seconds, found 6 clusters. [2026-01-20 07:07:15,212] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-20 07:07:15,249] INFO spindle_dev.index: Finished block 9 in 0.04 seconds, found 5 clusters. [2026-01-20 07:07:15,249] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-20 07:07:15,281] INFO spindle_dev.index: Finished block 10 in 0.03 seconds, found 4 clusters. [2026-01-20 07:07:15,282] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-20 07:07:15,314] INFO spindle_dev.index: Finished block 11 in 0.03 seconds, found 4 clusters. [2026-01-20 07:07:15,315] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-20 07:07:15,347] INFO spindle_dev.index: Finished block 12 in 0.03 seconds, found 4 clusters. [2026-01-20 07:07:15,348] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-20 07:07:15,384] INFO spindle_dev.index: Finished block 13 in 0.04 seconds, found 5 clusters. [2026-01-20 07:07:15,385] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-20 07:07:15,417] INFO spindle_dev.index: Finished block 14 in 0.03 seconds, found 4 clusters. [2026-01-20 07:07:15,418] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-20 07:07:15,446] INFO spindle_dev.index: Finished block 15 in 0.03 seconds, found 3 clusters. [2026-01-20 07:07:15,447] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-20 07:07:15,487] INFO spindle_dev.index: Finished block 16 in 0.04 seconds, found 4 clusters. [2026-01-20 07:07:15,488] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-20 07:07:15,529] INFO spindle_dev.index: Finished block 17 in 0.04 seconds, found 4 clusters. [2026-01-20 07:07:15,530] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-20 07:07:15,565] INFO spindle_dev.index: Finished block 18 in 0.04 seconds, found 4 clusters. [2026-01-20 07:07:15,566] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-20 07:07:15,609] INFO spindle_dev.index: Finished block 19 in 0.04 seconds, found 4 clusters. [2026-01-20 07:07:15,610] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-20 07:07:15,640] INFO spindle_dev.index: Finished block 20 in 0.03 seconds, found 3 clusters. [2026-01-20 07:07:15,640] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-20 07:07:15,666] INFO spindle_dev.index: Finished block 21 in 0.03 seconds, found 2 clusters. [2026-01-20 07:07:15,667] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-20 07:07:15,692] INFO spindle_dev.index: Finished block 22 in 0.03 seconds, found 2 clusters. [2026-01-20 07:07:15,693] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-20 07:07:15,719] INFO spindle_dev.index: Finished block 23 in 0.03 seconds, found 2 clusters. [2026-01-20 07:07:15,719] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-20 07:07:15,880] INFO spindle_dev.index: Finished block 24 in 0.16 seconds, found 6 clusters. [2026-01-20 07:07:15,881] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-20 07:07:15,916] INFO spindle_dev.index: Finished block 25 in 0.04 seconds, found 3 clusters. [2026-01-20 07:07:15,917] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-20 07:07:16,102] INFO spindle_dev.index: Finished block 26 in 0.19 seconds, found 5 clusters. [2026-01-20 07:07:16,103] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-20 07:07:16,103] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-20 07:07:16,103] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-20 07:07:16,103] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-20 07:07:16,104] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-20 07:07:16,111] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-20 07:07:16,112] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-20 07:07:16,112] INFO spindle_dev.index: Processing cluster 2 [2026-01-20 07:07:16,113] INFO spindle_dev.index: Building SPD index with epsilon=7.253261245720598 [2026-01-20 07:07:16,113] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-20 07:07:16,122] INFO spindle_dev.index: Cluster 2: 237 SPDs, 29 blocks [2026-01-20 07:07:16,122] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-20 07:07:16,132] INFO spindle_dev.index: Finished block 0 in 0.01 seconds, found 1 clusters. [2026-01-20 07:07:16,132] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-20 07:07:16,144] INFO spindle_dev.index: Finished block 1 in 0.01 seconds, found 3 clusters. [2026-01-20 07:07:16,144] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-20 07:07:16,155] INFO spindle_dev.index: Finished block 2 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:16,155] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-20 07:07:16,168] INFO spindle_dev.index: Finished block 3 in 0.01 seconds, found 3 clusters. [2026-01-20 07:07:16,168] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-20 07:07:16,179] INFO spindle_dev.index: Finished block 4 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:16,179] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-20 07:07:16,192] INFO spindle_dev.index: Finished block 5 in 0.01 seconds, found 3 clusters. [2026-01-20 07:07:16,192] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-20 07:07:16,205] INFO spindle_dev.index: Finished block 6 in 0.01 seconds, found 3 clusters. [2026-01-20 07:07:16,205] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-20 07:07:16,220] INFO spindle_dev.index: Finished block 7 in 0.01 seconds, found 4 clusters. [2026-01-20 07:07:16,220] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-20 07:07:16,238] INFO spindle_dev.index: Finished block 8 in 0.02 seconds, found 6 clusters. [2026-01-20 07:07:16,239] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-20 07:07:16,255] INFO spindle_dev.index: Finished block 9 in 0.02 seconds, found 5 clusters. [2026-01-20 07:07:16,256] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-20 07:07:16,271] INFO spindle_dev.index: Finished block 10 in 0.02 seconds, found 4 clusters. [2026-01-20 07:07:16,272] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-20 07:07:16,286] INFO spindle_dev.index: Finished block 11 in 0.01 seconds, found 4 clusters. [2026-01-20 07:07:16,287] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-20 07:07:16,298] INFO spindle_dev.index: Finished block 12 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:16,298] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-20 07:07:16,313] INFO spindle_dev.index: Finished block 13 in 0.01 seconds, found 4 clusters. [2026-01-20 07:07:16,314] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-20 07:07:16,334] INFO spindle_dev.index: Finished block 14 in 0.02 seconds, found 7 clusters. [2026-01-20 07:07:16,335] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-20 07:07:16,348] INFO spindle_dev.index: Finished block 15 in 0.01 seconds, found 3 clusters. [2026-01-20 07:07:16,348] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-20 07:07:16,363] INFO spindle_dev.index: Finished block 16 in 0.01 seconds, found 4 clusters. [2026-01-20 07:07:16,364] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-20 07:07:16,380] INFO spindle_dev.index: Finished block 17 in 0.02 seconds, found 5 clusters. [2026-01-20 07:07:16,381] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-20 07:07:16,398] INFO spindle_dev.index: Finished block 18 in 0.02 seconds, found 5 clusters. [2026-01-20 07:07:16,398] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-20 07:07:16,410] INFO spindle_dev.index: Finished block 19 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:16,410] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-20 07:07:16,423] INFO spindle_dev.index: Finished block 20 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:16,424] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-20 07:07:16,437] INFO spindle_dev.index: Finished block 21 in 0.01 seconds, found 3 clusters. [2026-01-20 07:07:16,438] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-20 07:07:16,451] INFO spindle_dev.index: Finished block 22 in 0.01 seconds, found 3 clusters. [2026-01-20 07:07:16,452] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-20 07:07:16,516] INFO spindle_dev.index: Finished block 23 in 0.06 seconds, found 4 clusters. [2026-01-20 07:07:16,517] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-20 07:07:16,544] INFO spindle_dev.index: Finished block 24 in 0.03 seconds, found 3 clusters. [2026-01-20 07:07:16,544] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-20 07:07:16,564] INFO spindle_dev.index: Finished block 25 in 0.02 seconds, found 3 clusters. [2026-01-20 07:07:16,565] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-20 07:07:16,583] INFO spindle_dev.index: Finished block 26 in 0.02 seconds, found 4 clusters. [2026-01-20 07:07:16,584] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-20 07:07:16,603] INFO spindle_dev.index: Finished block 27 in 0.02 seconds, found 4 clusters. [2026-01-20 07:07:16,603] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-20 07:07:16,624] INFO spindle_dev.index: Finished block 28 in 0.02 seconds, found 4 clusters. [2026-01-20 07:07:16,625] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-20 07:07:16,625] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-20 07:07:16,625] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-20 07:07:16,626] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-20 07:07:16,626] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-20 07:07:16,630] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-20 07:07:16,630] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-20 07:07:16,631] INFO spindle_dev.index: Processing cluster 3 [2026-01-20 07:07:16,631] INFO spindle_dev.index: Building SPD index with epsilon=7.55950504648754 [2026-01-20 07:07:16,631] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-20 07:07:16,638] INFO spindle_dev.index: Cluster 3: 156 SPDs, 24 blocks [2026-01-20 07:07:16,638] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-20 07:07:16,644] INFO spindle_dev.index: Finished block 0 in 0.01 seconds, found 1 clusters. [2026-01-20 07:07:16,644] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-20 07:07:16,650] INFO spindle_dev.index: Finished block 1 in 0.01 seconds, found 1 clusters. [2026-01-20 07:07:16,650] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-20 07:07:16,655] INFO spindle_dev.index: Finished block 2 in 0.01 seconds, found 1 clusters. [2026-01-20 07:07:16,656] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-20 07:07:16,661] INFO spindle_dev.index: Finished block 3 in 0.01 seconds, found 1 clusters. [2026-01-20 07:07:16,661] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-20 07:07:16,667] INFO spindle_dev.index: Finished block 4 in 0.01 seconds, found 1 clusters. [2026-01-20 07:07:16,668] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-20 07:07:16,674] INFO spindle_dev.index: Finished block 5 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:16,675] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-20 07:07:16,681] INFO spindle_dev.index: Finished block 6 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:16,682] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-20 07:07:16,689] INFO spindle_dev.index: Finished block 7 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:16,689] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-20 07:07:16,697] INFO spindle_dev.index: Finished block 8 in 0.01 seconds, found 3 clusters. [2026-01-20 07:07:16,697] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-20 07:07:16,704] INFO spindle_dev.index: Finished block 9 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:16,704] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-20 07:07:16,712] INFO spindle_dev.index: Finished block 10 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:16,712] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-20 07:07:16,719] INFO spindle_dev.index: Finished block 11 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:16,719] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-20 07:07:16,729] INFO spindle_dev.index: Finished block 12 in 0.01 seconds, found 4 clusters. [2026-01-20 07:07:16,729] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-20 07:07:16,737] INFO spindle_dev.index: Finished block 13 in 0.01 seconds, found 3 clusters. [2026-01-20 07:07:16,738] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-20 07:07:16,750] INFO spindle_dev.index: Finished block 14 in 0.01 seconds, found 6 clusters. [2026-01-20 07:07:16,751] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-20 07:07:16,762] INFO spindle_dev.index: Finished block 15 in 0.01 seconds, found 5 clusters. [2026-01-20 07:07:16,762] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-20 07:07:16,774] INFO spindle_dev.index: Finished block 16 in 0.01 seconds, found 4 clusters. [2026-01-20 07:07:16,775] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-20 07:07:16,785] INFO spindle_dev.index: Finished block 17 in 0.01 seconds, found 4 clusters. [2026-01-20 07:07:16,785] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-20 07:07:16,819] INFO spindle_dev.index: Finished block 18 in 0.03 seconds, found 5 clusters. [2026-01-20 07:07:16,819] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-20 07:07:16,882] INFO spindle_dev.index: Finished block 19 in 0.06 seconds, found 9 clusters. [2026-01-20 07:07:16,882] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-20 07:07:16,894] INFO spindle_dev.index: Finished block 20 in 0.01 seconds, found 5 clusters. [2026-01-20 07:07:16,894] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-20 07:07:16,917] INFO spindle_dev.index: Finished block 21 in 0.02 seconds, found 5 clusters. [2026-01-20 07:07:16,918] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-20 07:07:16,956] INFO spindle_dev.index: Finished block 22 in 0.04 seconds, found 7 clusters. [2026-01-20 07:07:16,956] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-20 07:07:16,966] INFO spindle_dev.index: Finished block 23 in 0.01 seconds, found 4 clusters. [2026-01-20 07:07:16,967] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-20 07:07:16,967] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-20 07:07:16,967] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-20 07:07:16,967] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-20 07:07:16,968] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-20 07:07:16,970] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-20 07:07:16,971] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-20 07:07:16,971] INFO spindle_dev.index: Processing cluster 4 [2026-01-20 07:07:16,971] INFO spindle_dev.index: Building SPD index with epsilon=6.14807850563238 [2026-01-20 07:07:16,971] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-20 07:07:16,973] INFO spindle_dev.index: Cluster 4: 37 SPDs, 22 blocks [2026-01-20 07:07:16,974] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-20 07:07:16,975] INFO spindle_dev.index: Finished block 0 in 0.00 seconds, found 1 clusters. [2026-01-20 07:07:16,976] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-20 07:07:16,978] INFO spindle_dev.index: Finished block 1 in 0.00 seconds, found 4 clusters. [2026-01-20 07:07:16,979] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-20 07:07:16,981] INFO spindle_dev.index: Finished block 2 in 0.00 seconds, found 2 clusters. [2026-01-20 07:07:16,981] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-20 07:07:16,984] INFO spindle_dev.index: Finished block 3 in 0.00 seconds, found 4 clusters. [2026-01-20 07:07:16,984] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-20 07:07:16,987] INFO spindle_dev.index: Finished block 4 in 0.00 seconds, found 4 clusters. [2026-01-20 07:07:16,987] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-20 07:07:16,990] INFO spindle_dev.index: Finished block 5 in 0.00 seconds, found 6 clusters. [2026-01-20 07:07:16,990] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-20 07:07:16,993] INFO spindle_dev.index: Finished block 6 in 0.00 seconds, found 5 clusters. [2026-01-20 07:07:16,993] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-20 07:07:16,997] INFO spindle_dev.index: Finished block 7 in 0.00 seconds, found 8 clusters. [2026-01-20 07:07:16,998] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-20 07:07:17,001] INFO spindle_dev.index: Finished block 8 in 0.00 seconds, found 7 clusters. [2026-01-20 07:07:17,001] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-20 07:07:17,005] INFO spindle_dev.index: Finished block 9 in 0.00 seconds, found 8 clusters. [2026-01-20 07:07:17,005] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-20 07:07:17,009] INFO spindle_dev.index: Finished block 10 in 0.00 seconds, found 7 clusters. [2026-01-20 07:07:17,009] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-20 07:07:17,013] INFO spindle_dev.index: Finished block 11 in 0.00 seconds, found 9 clusters. [2026-01-20 07:07:17,014] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-20 07:07:17,017] INFO spindle_dev.index: Finished block 12 in 0.00 seconds, found 6 clusters. [2026-01-20 07:07:17,017] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-20 07:07:17,021] INFO spindle_dev.index: Finished block 13 in 0.00 seconds, found 9 clusters. [2026-01-20 07:07:17,022] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-20 07:07:17,024] INFO spindle_dev.index: Finished block 14 in 0.00 seconds, found 3 clusters. [2026-01-20 07:07:17,024] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-20 07:07:17,028] INFO spindle_dev.index: Finished block 15 in 0.00 seconds, found 6 clusters. [2026-01-20 07:07:17,028] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-20 07:07:17,032] INFO spindle_dev.index: Finished block 16 in 0.00 seconds, found 4 clusters. [2026-01-20 07:07:17,033] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-20 07:07:17,046] INFO spindle_dev.index: Finished block 17 in 0.01 seconds, found 7 clusters. [2026-01-20 07:07:17,047] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-20 07:07:17,050] INFO spindle_dev.index: Finished block 18 in 0.00 seconds, found 5 clusters. [2026-01-20 07:07:17,051] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-20 07:07:17,060] INFO spindle_dev.index: Finished block 19 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:17,060] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-20 07:07:17,069] INFO spindle_dev.index: Finished block 20 in 0.01 seconds, found 2 clusters. [2026-01-20 07:07:17,069] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-20 07:07:17,074] INFO spindle_dev.index: Finished block 21 in 0.01 seconds, found 1 clusters. [2026-01-20 07:07:17,075] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-20 07:07:17,075] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-20 07:07:17,075] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-20 07:07:17,075] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-20 07:07:17,076] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-20 07:07:17,077] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-20 07:07:17,078] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-20 07:07:17,078] INFO spindle_dev.index: Processing cluster 5 [2026-01-20 07:07:17,079] INFO spindle_dev.index: Building SPD index with epsilon=3.819779091158857 [2026-01-20 07:07:17,079] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-20 07:07:17,080] INFO spindle_dev.index: Cluster 5: 21 SPDs, 31 blocks [2026-01-20 07:07:17,080] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-20 07:07:17,083] INFO spindle_dev.index: Finished block 0 in 0.00 seconds, found 8 clusters. [2026-01-20 07:07:17,083] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-20 07:07:17,086] INFO spindle_dev.index: Finished block 1 in 0.00 seconds, found 13 clusters. [2026-01-20 07:07:17,087] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-20 07:07:17,090] INFO spindle_dev.index: Finished block 2 in 0.00 seconds, found 14 clusters. [2026-01-20 07:07:17,090] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-20 07:07:17,094] INFO spindle_dev.index: Finished block 3 in 0.00 seconds, found 17 clusters. [2026-01-20 07:07:17,094] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-20 07:07:17,099] INFO spindle_dev.index: Finished block 4 in 0.00 seconds, found 19 clusters. [2026-01-20 07:07:17,099] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-20 07:07:17,104] INFO spindle_dev.index: Finished block 5 in 0.00 seconds, found 21 clusters. [2026-01-20 07:07:17,104] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-20 07:07:17,108] INFO spindle_dev.index: Finished block 6 in 0.00 seconds, found 17 clusters. [2026-01-20 07:07:17,108] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-20 07:07:17,112] INFO spindle_dev.index: Finished block 7 in 0.00 seconds, found 20 clusters. [2026-01-20 07:07:17,113] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-20 07:07:17,116] INFO spindle_dev.index: Finished block 8 in 0.00 seconds, found 15 clusters. [2026-01-20 07:07:17,117] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-20 07:07:17,120] INFO spindle_dev.index: Finished block 9 in 0.00 seconds, found 13 clusters. [2026-01-20 07:07:17,120] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-20 07:07:17,123] INFO spindle_dev.index: Finished block 10 in 0.00 seconds, found 9 clusters. [2026-01-20 07:07:17,123] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-20 07:07:17,127] INFO spindle_dev.index: Finished block 11 in 0.00 seconds, found 13 clusters. [2026-01-20 07:07:17,127] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-20 07:07:17,129] INFO spindle_dev.index: Finished block 12 in 0.00 seconds, found 6 clusters. [2026-01-20 07:07:17,129] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-20 07:07:17,131] INFO spindle_dev.index: Finished block 13 in 0.00 seconds, found 2 clusters. [2026-01-20 07:07:17,131] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-20 07:07:17,133] INFO spindle_dev.index: Finished block 14 in 0.00 seconds, found 1 clusters. [2026-01-20 07:07:17,133] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-20 07:07:17,136] INFO spindle_dev.index: Finished block 15 in 0.00 seconds, found 2 clusters. [2026-01-20 07:07:17,136] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-20 07:07:17,138] INFO spindle_dev.index: Finished block 16 in 0.00 seconds, found 2 clusters. [2026-01-20 07:07:17,139] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-20 07:07:17,141] INFO spindle_dev.index: Finished block 17 in 0.00 seconds, found 4 clusters. [2026-01-20 07:07:17,141] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-20 07:07:17,142] INFO spindle_dev.index: Finished block 18 in 0.00 seconds, found 2 clusters. [2026-01-20 07:07:17,143] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-20 07:07:17,145] INFO spindle_dev.index: Finished block 19 in 0.00 seconds, found 5 clusters. [2026-01-20 07:07:17,145] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-20 07:07:17,146] INFO spindle_dev.index: Finished block 20 in 0.00 seconds, found 2 clusters. [2026-01-20 07:07:17,147] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-20 07:07:17,150] INFO spindle_dev.index: Finished block 21 in 0.00 seconds, found 9 clusters. [2026-01-20 07:07:17,150] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-20 07:07:17,152] INFO spindle_dev.index: Finished block 22 in 0.00 seconds, found 3 clusters. [2026-01-20 07:07:17,153] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-20 07:07:17,155] INFO spindle_dev.index: Finished block 23 in 0.00 seconds, found 8 clusters. [2026-01-20 07:07:17,155] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-20 07:07:17,157] INFO spindle_dev.index: Finished block 24 in 0.00 seconds, found 3 clusters. [2026-01-20 07:07:17,158] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-20 07:07:17,160] INFO spindle_dev.index: Finished block 25 in 0.00 seconds, found 10 clusters. [2026-01-20 07:07:17,161] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-20 07:07:17,162] INFO spindle_dev.index: Finished block 26 in 0.00 seconds, found 4 clusters. [2026-01-20 07:07:17,163] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-20 07:07:17,165] INFO spindle_dev.index: Finished block 27 in 0.00 seconds, found 5 clusters. [2026-01-20 07:07:17,165] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-20 07:07:17,167] INFO spindle_dev.index: Finished block 28 in 0.00 seconds, found 3 clusters. [2026-01-20 07:07:17,167] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-20 07:07:17,168] INFO spindle_dev.index: Finished block 29 in 0.00 seconds, found 2 clusters. [2026-01-20 07:07:17,169] INFO spindle_dev.index: Using epsilon-net clustering for block 30 [2026-01-20 07:07:17,170] INFO spindle_dev.index: Finished block 30 in 0.00 seconds, found 3 clusters. [2026-01-20 07:07:17,171] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-20 07:07:17,171] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-20 07:07:17,171] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-20 07:07:17,172] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-20 07:07:17,172] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-20 07:07:17,174] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-20 07:07:17,174] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances.
In [26]:
search_cfg = search.SearchConfig(max_results=2, debug=False, max_failed_starts=5, max_failed_paths=10)
In [27]:
test_results = test.run_sanity_search(data, dag_dict, config, search_cfg, max_queries=2000, skip_baseline=True)
[2026-01-20 07:07:25,850] INFO spindle_dev.test: Built ground-truth paths for 872 SPDs in cluster 0 across 28 blocks. [2026-01-20 07:07:26,124] INFO spindle_dev.test: Built ground-truth paths for 526 SPDs in cluster 1 across 27 blocks. [2026-01-20 07:07:26,187] INFO spindle_dev.test: Built ground-truth paths for 237 SPDs in cluster 2 across 29 blocks. [2026-01-20 07:07:26,211] INFO spindle_dev.test: Built ground-truth paths for 156 SPDs in cluster 3 across 24 blocks. [2026-01-20 07:07:26,214] INFO spindle_dev.test: Built ground-truth paths for 37 SPDs in cluster 4 across 22 blocks. [2026-01-20 07:07:26,216] INFO spindle_dev.test: Built ground-truth paths for 21 SPDs in cluster 5 across 31 blocks. [2026-01-20 07:07:26,216] INFO spindle_dev.test: Created ground-truth paths for all clusters. [2026-01-20 07:07:26,217] INFO spindle_dev.test: Search config: SearchConfig(max_results=2, max_failed_starts=5, max_failed_paths=10, total_paths_limit=1000, deterministic=DeterministicConfig(seed=0), debug=False) [2026-01-20 07:07:26,218] INFO spindle_dev.test: Running sanity search with 1849 queries. 100%|██████████| 1849/1849 [00:29<00:00, 61.69it/s] [2026-01-20 07:07:56,194] INFO spindle_dev.test: Sanity search: 1849 queries, 1849 exact path matches, 1849 leaf matches, mean search_time=0.0155s
In [24]:
fig = plotting.visualize_block_dag_sankey_scaled(dag_dict[2].nodes, height=800, thickness=20.0)
fig
In [28]:
import pandas as pd
def get_dag_stats(dag_dict):
dag_stats = []
for cluster_id, cluster_dag in dag_dict.items():
block_to_node = cluster_dag.block_to_node_indices
for node in cluster_dag.nodes:
num_spds = len(node.metadata.members)
block_size = node.metadata.mean.shape[0]
block_id = node.block_index
radius = node.metadata.radius
#print(f"Cluster_id {cluster_id} Node id {node.global_node_id} Block {block_id}: {num_spds} spds")
dag_stats.append({
"cluster_id": cluster_id,
"node_id": node.global_node_id,
"block_id": block_id,
"block_cluster_id": node.block_cluster_id,
"num_spds": num_spds,
"radius": radius,
"block_size": block_size
})
dag_stats_df = pd.DataFrame(dag_stats)
return dag_stats_df
In [29]:
dag_stats_df = get_dag_stats(dag_dict)
In [30]:
score_list = []
for cluster_id in set(data.labels):
cluster_dag = dag_dict[cluster_id]
cluster_score = index.score_nodes_from_a_cluster(data, cluster_id, dag_dict, take_representative_mean=True)
score_list.extend(cluster_score)
100%|██████████| 87/87 [00:00<00:00, 304.09it/s] 100%|██████████| 104/104 [00:00<00:00, 314.05it/s] 100%|██████████| 102/102 [00:00<00:00, 528.78it/s] 100%|██████████| 81/81 [00:00<00:00, 189.76it/s] 100%|██████████| 110/110 [00:00<00:00, 328.16it/s] 100%|██████████| 255/255 [00:00<00:00, 880.33it/s]
In [31]:
node_scores = []
for node_score in score_list:
node_scores.append({
"cluster_id": node_score["cluster_id"],
"block_id": node_score["block_id"],
"node_id": node_score["node_id"],
"node_score": node_score["node_score"],
"num_spds": node_score["num_spds"],
"block_size": node_score["block_size"],
"E": node_score["E"],
"Q": node_score["Q"],
"S": node_score["S"],
"num_modules": len(node_score["modules"])
})
node_score_df = pd.DataFrame(node_scores)
In [32]:
node_score_df.sort_values(by='node_score', ascending=False).head(20)
Out[32]:
| cluster_id | block_id | node_id | node_score | num_spds | block_size | E | Q | S | num_modules | |
|---|---|---|---|---|---|---|---|---|---|---|
| 351 | 3 | 19 | 58 | 0.342765 | 2 | 58 | 2.002731 | 0.262735 | 0.348588 | 4 |
| 479 | 4 | 19 | 105 | 0.267199 | 36 | 44 | 1.682971 | 0.301508 | 0.473429 | 4 |
| 54 | 0 | 20 | 54 | 0.245330 | 6 | 56 | 2.982897 | 0.172531 | 0.523302 | 10 |
| 471 | 4 | 17 | 97 | 0.239994 | 1 | 55 | 1.694165 | 0.273254 | 0.481585 | 5 |
| 349 | 3 | 19 | 56 | 0.234528 | 2 | 58 | 1.849325 | 0.203176 | 0.375823 | 7 |
| 187 | 1 | 26 | 100 | 0.220137 | 1 | 54 | 1.770100 | 0.206593 | 0.398025 | 7 |
| 369 | 3 | 22 | 76 | 0.219125 | 3 | 41 | 2.449950 | 0.184739 | 0.515857 | 3 |
| 470 | 4 | 17 | 96 | 0.218723 | 1 | 55 | 3.007032 | 0.182812 | 0.602124 | 4 |
| 180 | 1 | 24 | 93 | 0.214378 | 1 | 47 | 2.094271 | 0.195042 | 0.475172 | 6 |
| 366 | 3 | 22 | 73 | 0.214279 | 1 | 41 | 2.327545 | 0.192251 | 0.521138 | 6 |
| 465 | 4 | 16 | 91 | 0.212545 | 2 | 21 | 1.730564 | 0.249610 | 0.507961 | 2 |
| 467 | 4 | 17 | 93 | 0.207663 | 11 | 55 | 1.399861 | 0.235738 | 0.370723 | 7 |
| 347 | 3 | 19 | 54 | 0.202500 | 2 | 58 | 2.895693 | 0.139766 | 0.499658 | 4 |
| 352 | 3 | 19 | 59 | 0.200235 | 3 | 58 | 2.312101 | 0.162095 | 0.465731 | 4 |
| 177 | 1 | 24 | 90 | 0.195437 | 2 | 47 | 1.964387 | 0.172840 | 0.424383 | 5 |
| 60 | 0 | 20 | 60 | 0.195213 | 3 | 56 | 2.608806 | 0.145667 | 0.486308 | 4 |
| 179 | 1 | 24 | 92 | 0.193402 | 1 | 47 | 1.880612 | 0.169196 | 0.392191 | 4 |
| 341 | 3 | 18 | 48 | 0.187174 | 4 | 39 | 1.817072 | 0.207535 | 0.503662 | 5 |
| 59 | 0 | 20 | 59 | 0.185682 | 1 | 56 | 2.075356 | 0.162446 | 0.449234 | 5 |
| 348 | 3 | 19 | 55 | 0.179555 | 39 | 58 | 1.421501 | 0.224009 | 0.436125 | 6 |
In [33]:
importlib.reload(plotting)
Out[33]:
<module 'spindle_dev.plotting' from '/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/plotting.py'>
In [120]:
#id = 513
id = 351
scores_846, fig_846 = plotting.plot_module_heatmap_plus_spatial(
id,
score_list,
node_score_df,
dag_stats_df,
dag_dict,
data,
adata,
spot_size=0.2
)
Creating combined figure for cluster 3, node 58, block 19
In [ ]:
In [121]:
fig_846.savefig(f"{result_dir}/{id}_module_0_1_enrichment.png", dpi=300)
In [122]:
from spindle_dev import go_score
In [123]:
out_846 = go_score.enrich_modules_with_gseapy(score_list[id]['modules'])
Module 0: enrichment completed with 604 terms Module 1: enrichment completed with 351 terms Module 2: enrichment completed with 587 terms Module 3: enrichment completed with 138 terms
In [124]:
importlib.reload(go_score)
Out[124]:
<module 'spindle_dev.go_score' from '/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/go_score.py'>
In [126]:
for module_id, module_genes, res in out_846:
if module_id == 1:
print(module_genes)
fig, axes = go_score.plot_module_enrichment_libraries_2(
module_id=module_id,
module_genes=module_genes,
res=res,
top_n=7,
ncols=2,
bar_color="#D41515EE", # pick your color
tick_fontsize=10,
label_fontsize=10,
title_fontsize=8,
compact_height_per_term=0.2,
min_fig_h=8,
plot_kind="dot",
size_range=(20, 200),
use_constrained_layout=False,
save_dir=f"{result_dir}/go_enrichment/enrichment_{id}"
)
if fig is not None:
plt.show()
['ACTA2', 'ADAMTS1', 'AMY2A', 'CNN1', 'CRISPLD2', 'DPT', 'DST', 'GEM', 'GPC1', 'LTBP2', 'MEDAG', 'MYH11', 'MYLK', 'NTN4', 'PDGFRB', 'SLC26A2', 'TBX3']
/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/go_score.py:519: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect. # fig.subplots_adjust(left=left_margin, wspace=wspace, hspace=hspace, top=0.90)
In [39]:
# Create a dictionary of tile_id to block_id
# Clusters do not share tile ids across them
from collections import defaultdict
tile_to_block_dict = {}
cluster_to_tile = defaultdict(set)
for cluster_id, cluster_dag in dag_dict.items():
block_to_node = cluster_dag.block_to_node_indices
for node in cluster_dag.nodes:
member_tile_ids = [tid for tid,_ in node.metadata.members]
for tile_id in member_tile_ids:
if not tile_id in tile_to_block_dict:
tile_to_block_dict[tile_id] = {}
tile_to_block_dict[tile_id][node.block_index] = node.block_cluster_id
cluster_to_tile[cluster_id].add(tile_id)
In [40]:
def get_spot_score_dict(block_id, cluster_id, tile_to_block_dict, grid=False):
spot_score = {}
tiles_of_cluster = cluster_to_tile[cluster_id]
for tid in tiles_of_cluster:
if tid not in tile_to_block_dict:
continue
c = tile_to_block_dict[tid][block_id]
t = data.metadata['tiles'][tid]
for id in t.idx:
spot_score[id] = c
fig, ax = plotting.plot_spot_module_scores(spot_score, tiles, adata, color='discrete', grid=grid)
return fig, ax
In [48]:
fig, ax = get_spot_score_dict(19, 4, tile_to_block_dict, grid=False)
In [ ]:
sc.pl.spatial(adata,color=['ARFGEF3'], spot_size=20, cmap='Reds')
/tmp/ipykernel_442280/2723282100.py:1: FutureWarning: Use `squidpy.pl.spatial_scatter` instead. sc.pl.spatial(adata,color=['ARFGEF3'], spot_size=10, cmap='Reds')
In [56]:
res.columns
Out[56]:
Index(['Gene_set', 'Term', 'Overlap', 'P-value', 'Adjusted P-value',
'Old P-value', 'Old Adjusted P-value', 'Odds Ratio', 'Combined Score',
'Genes'],
dtype='object')
In [ ]: