In [2]:
# Set up sys.path so that 'src/spindle_dev' is importable as 'spindle_dev'
import sys
import importlib
from pathlib import Path
project_root = '/data/sarkar_lab/Projects/spindle_dev'
src_path = Path(project_root) / 'src'
if str(src_path) not in sys.path:
sys.path.insert(0, str(src_path))
import spindle_dev
# Reload to pick up code changes without restarting the kernel
importlib.reload(spindle_dev)
Out[2]:
<module 'spindle_dev' from '/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/__init__.py'>
In [3]:
import scanpy as sc
In [4]:
import glob
In [5]:
h5ad_files = glob.glob("/data/sarkar_lab/insitupy_demo_data_xenium/*.h5ad")
In [6]:
h5ad_files
Out[6]:
['/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_skin_melanoma.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_brain_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_kidney_nondiseased.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lung_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lymph_node.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lymph_node_5k.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_pancreatic_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_breast_cancer.h5ad']
In [7]:
adata = sc.read_h5ad('/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_kidney_nondiseased.h5ad')
In [8]:
from pathlib import Path
import spindle_dev
import spindle_dev.metrics as metrics
import spindle_dev.index as index
import spindle_dev.preprocessing as preprocessing
import spindle_dev.plotting as plotting
import spindle_dev.test as test
import spindle_dev.search as search
import spindle_dev.typing as typing
# Reload to pick up code changes without restarting the kernel
import time
import scanpy as sc
import glob
import pandas as pd
import numpy as np
from concurrent.futures import ThreadPoolExecutor, as_completed
from joblib import Parallel, delayed
import argparse
In [9]:
result_dir = "/data/sarkar_lab/Projects/spindle_dev/results/hkidney_10X/"
Path(result_dir).mkdir(parents=True, exist_ok=True)
In [10]:
all_genes = True
resolution=0.5
min_final_size=10
start_time = time.time()
coords = adata.obsm["spatial"]
tiles = preprocessing.build_quadtree_tiles(coords, max_pts=100, min_side=0.0, max_depth=40)
# remove tiles with less than 5 spots
tiles = [tile for tile in tiles if len(tile.idx) >= 5]
tiles = preprocessing.reindex_tiles(tiles)
num_genes = adata.n_vars
genes_work, gene_idx = spindle_dev.preprocessing.topvar_genes(adata, G=num_genes)
tile_covs = spindle_dev.preprocessing.build_tile_covs_full_serial(adata, tiles, gene_idx, eps=1e-6)
data = index.ProcessedData(tiles, tile_covs, genes_work, adata.n_obs)
if 'pca' not in data.latent:
data.reduce_dim(num_pca_components=30, n_components=2, do_umap=True)
data.cluster_spds(cluster_distance="tree", cluster_method="leiden", resolution=resolution)
data.assign_label_to_spots()
data.get_corr_mean_by_cluster()
out_dict = data.get_adaptive_runs(find_blocks=True, with_size_guard=True,min_final_size=min_final_size,max_final_size=100)
epsilon_block_wise_dict = {}
epsilon_dict = {}
for cluster_id in set(data.labels):
eps_per_block, eps_elbow_per_block, eps = index.choose_adaptive_epsilons(data, cluster_id, k_target_per_block=64)
epsilon_block_wise_dict[int(cluster_id)] = eps_elbow_per_block
epsilon_dict[int(cluster_id)] = eps
[2026-01-21 04:15:40,014] INFO spindle_dev.index: Clustering SPD-s using 'tree' distance. [2026-01-21 04:15:40,113] INFO spindle_dev.index: Building ultrametric features from SPD matrices. [2026-01-21 04:15:52,765] INFO spindle_dev.index: Computing latent features from the tree representations. [2026-01-21 04:15:53,798] INFO spindle_dev.index: Reducing latent features to 30 dimensions using PCA. [2026-01-21 04:16:04,801] INFO spindle_dev.index: Explained variance ratios by PCA components: [0.02053706 0.01375599 0.00970807 0.00620443 0.00564134 0.00534054 0.00414861 0.00382235 0.00377741 0.00335041 0.00304303 0.00294533 0.00263028 0.00260522 0.002546 0.00247329 0.00238618 0.00235676 0.00221383 0.00215483 0.0021273 0.00207967 0.0020493 0.002041 0.00201291 0.00199242 0.00198083 0.00195477 0.00194887 0.00190091] [2026-01-21 04:16:04,803] INFO spindle_dev.index: Reducing latent features to 2 dimensions using UMAP. /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/umap/umap_.py:1952: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism. warn( [2026-01-21 04:16:18,812] INFO spindle_dev.index: Clustering SPD-s using 'tree' distance. [2026-01-21 04:16:18,812] INFO spindle_dev.index: Clustering SPD matrices using Leiden clustering with resolution 0.50. [2026-01-21 04:16:19,204] INFO spindle_dev.index: Since clustering method is tree, I am going to find global order per cluster [2026-01-21 04:16:19,204] INFO spindle_dev.index: Finding consensus tree for cluster 0 [2026-01-21 04:16:19,470] INFO spindle_dev.index: Finding consensus tree for cluster 1 [2026-01-21 04:16:19,719] INFO spindle_dev.index: Finding consensus tree for cluster 2 [2026-01-21 04:16:19,952] INFO spindle_dev.index: Finding consensus tree for cluster 3 [2026-01-21 04:16:20,052] INFO spindle_dev.index: Finding consensus tree for cluster 4 [2026-01-21 04:16:21,045] INFO spindle_dev.index: Computing mean correlation matrix for cluster 0 [2026-01-21 04:16:22,627] INFO spindle_dev.index: Computing mean correlation matrix for cluster 1 [2026-01-21 04:16:24,151] INFO spindle_dev.index: Computing mean correlation matrix for cluster 2 [2026-01-21 04:16:25,586] INFO spindle_dev.index: Computing mean correlation matrix for cluster 3 [2026-01-21 04:16:26,015] INFO spindle_dev.index: Computing mean correlation matrix for cluster 4 [2026-01-21 04:16:26,263] INFO spindle_dev.index: Finding adaptive block runs for cluster 0 [2026-01-21 04:16:26,668] INFO spindle_dev.index: Chose t=0.872736312432029 resulting in 242 blocks instead of 184 blocks would have gotten by default [2026-01-21 04:16:26,713] INFO spindle_dev.index: Final block runs for cluster 0: 28 blocks. [2026-01-21 04:16:26,714] INFO spindle_dev.index: Finding adaptive block runs for cluster 1 [2026-01-21 04:16:27,109] INFO spindle_dev.index: Chose t=0.8723687514376944 resulting in 267 blocks instead of 222 blocks would have gotten by default [2026-01-21 04:16:27,165] INFO spindle_dev.index: Final block runs for cluster 1: 30 blocks. [2026-01-21 04:16:27,165] INFO spindle_dev.index: Finding adaptive block runs for cluster 2 [2026-01-21 04:16:27,556] INFO spindle_dev.index: Chose t=0.8535490370593214 resulting in 297 blocks instead of 240 blocks would have gotten by default [2026-01-21 04:16:27,623] INFO spindle_dev.index: Final block runs for cluster 2: 30 blocks. [2026-01-21 04:16:27,624] INFO spindle_dev.index: Finding adaptive block runs for cluster 3 [2026-01-21 04:16:28,014] INFO spindle_dev.index: Chose t=0.900984862397922 resulting in 212 blocks instead of 186 blocks would have gotten by default [2026-01-21 04:16:28,052] INFO spindle_dev.index: Final block runs for cluster 3: 26 blocks. [2026-01-21 04:16:28,053] INFO spindle_dev.index: Finding adaptive block runs for cluster 4 [2026-01-21 04:16:28,444] INFO spindle_dev.index: Chose t=0.8689389794831175 resulting in 297 blocks instead of 243 blocks would have gotten by default [2026-01-21 04:16:28,511] INFO spindle_dev.index: Final block runs for cluster 4: 31 blocks.
In [11]:
import matplotlib.pylab as plt
from matplotlib.patches import Rectangle
from matplotlib.collections import PatchCollection
In [12]:
import numpy as np
palette = sc.pl.palettes.default_20
point_colors = [palette[lab] for lab in data.spot_label.values()]
labels = data.labels
indices = np.array([int(i) for i in data.spot_label.keys()])
coords = adata.obsm['spatial']
plt.scatter(coords[indices,0], coords[indices,1], s=1, c=point_colors, lw=0, alpha=0.8)
patches = []
for t in tiles:
bbox = t.bbox if hasattr(t, "bbox") else t["bbox"] # (xmin,ymin,xmax,ymax)
x0, y0, x1, y1 = bbox
patches.append(Rectangle((x0, y0), x1 - x0, y1 - y0, fill=False))
pc = PatchCollection(patches, match_original=True, linewidths=0.2, edgecolors='b', alpha=0.3)
# add collection
ax = plt.gca()
ax.add_collection(pc)
ax.invert_yaxis()
ax.axis('off')
plt.savefig(f"{result_dir}/spatial_plot_with_cluster.png", dpi=300)
plt.show()
In [13]:
# Create colors for clusters
import numpy as np
import matplotlib.pyplot as plt
n_clusters = len(set(data.labels))
# Use Scanpy's default_20 palette for consistency
palette = sc.pl.palettes.default_20
cluster_colors = [palette[lab] for lab in data.labels]
plt.scatter(data.latent['umap'][:,0], data.latent['umap'][:,1], s=6, lw=0, alpha=0.8, c=cluster_colors)
# Write label on the plot
# with backgrond circle for better visibility
# and bold font
for i in range(n_clusters):
cluster_points = data.latent['umap'][data.labels == i]
if len(cluster_points) == 0:
continue
x_mean = np.mean(cluster_points[:,0])
y_mean = np.mean(cluster_points[:,1])
plt.text(x_mean, y_mean, str(i), color='black', fontsize=10, ha='center', va='center', fontweight='bold',
bbox=dict(facecolor='white', edgecolor='black', boxstyle='circle,pad=0.5', alpha=0.7))
# take away top and right border
ax = plt.gca()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
# remove axis ticks
ax.set_xticks([])
ax.set_yticks([])
plt.savefig(f"{result_dir}/umap_plot_with_cluster_labels.png", dpi=300)
plt.show()
In [14]:
epsilon_block_wise_dict = {}
epsilon_dict = {}
for cluster_id in set(data.labels):
eps_per_block, eps_elbow_per_block, eps = index.choose_adaptive_epsilons(data, cluster_id, k_target_per_block=64)
epsilon_block_wise_dict[int(cluster_id)] = eps_elbow_per_block
epsilon_dict[int(cluster_id)] = eps
In [15]:
epsilon_dict
Out[15]:
{0: np.float64(8.260153779446961),
1: np.float64(8.066662756016948),
2: np.float64(8.17919796610604),
3: np.float64(8.007105696219579),
4: np.float64(7.326412259271399)}
In [16]:
config = typing.IndexConfig()
config.epsilon_dict = epsilon_dict
config.epsilon_block_wise_dict = epsilon_block_wise_dict
config.threshold_type = 'block-wise'
config.kmean_method = 'epsilon_net'
In [17]:
# Create index
dag_dict, stat, dist_list = index.index_spds(data, config=config)
[2026-01-21 04:16:46,140] INFO spindle_dev.index: Processing cluster 0 [2026-01-21 04:16:46,140] INFO spindle_dev.index: Building SPD index with epsilon=8.260153779446961 [2026-01-21 04:16:46,142] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:16:46,197] INFO spindle_dev.index: Cluster 0: 743 SPDs, 28 blocks [2026-01-21 04:16:46,198] INFO spindle_dev.index: Using epsilon-net clustering for block 0
[2026-01-21 04:16:46,221] INFO spindle_dev.index: Finished block 0 in 0.02 seconds, found 1 clusters. [2026-01-21 04:16:46,222] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:16:46,245] INFO spindle_dev.index: Finished block 1 in 0.02 seconds, found 1 clusters. [2026-01-21 04:16:46,245] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:16:46,275] INFO spindle_dev.index: Finished block 2 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:46,275] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:16:46,305] INFO spindle_dev.index: Finished block 3 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:46,305] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:16:46,342] INFO spindle_dev.index: Finished block 4 in 0.04 seconds, found 2 clusters. [2026-01-21 04:16:46,343] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:16:46,379] INFO spindle_dev.index: Finished block 5 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:46,379] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:16:46,410] INFO spindle_dev.index: Finished block 6 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:46,410] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:16:46,447] INFO spindle_dev.index: Finished block 7 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:46,448] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:16:46,479] INFO spindle_dev.index: Finished block 8 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:46,479] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:16:46,517] INFO spindle_dev.index: Finished block 9 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:46,517] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:16:46,555] INFO spindle_dev.index: Finished block 10 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:46,556] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:16:46,588] INFO spindle_dev.index: Finished block 11 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:46,588] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:16:46,626] INFO spindle_dev.index: Finished block 12 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:46,627] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:16:46,660] INFO spindle_dev.index: Finished block 13 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:46,661] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:16:46,710] INFO spindle_dev.index: Finished block 14 in 0.05 seconds, found 5 clusters. [2026-01-21 04:16:46,711] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:16:46,755] INFO spindle_dev.index: Finished block 15 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:46,755] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:16:46,800] INFO spindle_dev.index: Finished block 16 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:46,801] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:16:46,845] INFO spindle_dev.index: Finished block 17 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:46,845] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:16:46,890] INFO spindle_dev.index: Finished block 18 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:46,891] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:16:46,937] INFO spindle_dev.index: Finished block 19 in 0.05 seconds, found 4 clusters. [2026-01-21 04:16:46,937] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:16:46,982] INFO spindle_dev.index: Finished block 20 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:46,983] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:16:47,022] INFO spindle_dev.index: Finished block 21 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:47,023] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:16:47,121] INFO spindle_dev.index: Finished block 22 in 0.10 seconds, found 2 clusters. [2026-01-21 04:16:47,122] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:16:47,178] INFO spindle_dev.index: Finished block 23 in 0.06 seconds, found 3 clusters. [2026-01-21 04:16:47,179] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:16:47,228] INFO spindle_dev.index: Finished block 24 in 0.05 seconds, found 4 clusters. [2026-01-21 04:16:47,228] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:16:47,282] INFO spindle_dev.index: Finished block 25 in 0.05 seconds, found 3 clusters. [2026-01-21 04:16:47,283] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:16:47,667] INFO spindle_dev.index: Finished block 26 in 0.38 seconds, found 39 clusters. [2026-01-21 04:16:47,668] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:16:47,750] INFO spindle_dev.index: Finished block 27 in 0.08 seconds, found 3 clusters. [2026-01-21 04:16:47,750] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:16:47,751] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:16:47,751] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:16:47,751] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:16:47,752] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:16:48,074] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:16:48,075] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:16:48,075] INFO spindle_dev.index: Processing cluster 1 [2026-01-21 04:16:48,076] INFO spindle_dev.index: Building SPD index with epsilon=8.066662756016948 [2026-01-21 04:16:48,076] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:16:48,126] INFO spindle_dev.index: Cluster 1: 735 SPDs, 30 blocks [2026-01-21 04:16:48,127] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:16:48,153] INFO spindle_dev.index: Finished block 0 in 0.03 seconds, found 1 clusters. [2026-01-21 04:16:48,153] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:16:48,176] INFO spindle_dev.index: Finished block 1 in 0.02 seconds, found 1 clusters. [2026-01-21 04:16:48,177] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:16:48,201] INFO spindle_dev.index: Finished block 2 in 0.02 seconds, found 1 clusters. [2026-01-21 04:16:48,201] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:16:48,230] INFO spindle_dev.index: Finished block 3 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:48,231] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:16:48,254] INFO spindle_dev.index: Finished block 4 in 0.02 seconds, found 1 clusters. [2026-01-21 04:16:48,255] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:16:48,289] INFO spindle_dev.index: Finished block 5 in 0.03 seconds, found 3 clusters. [2026-01-21 04:16:48,290] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:16:48,319] INFO spindle_dev.index: Finished block 6 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:48,320] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:16:48,349] INFO spindle_dev.index: Finished block 7 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:48,350] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:16:48,391] INFO spindle_dev.index: Finished block 8 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:48,391] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:16:48,428] INFO spindle_dev.index: Finished block 9 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:48,429] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:16:48,468] INFO spindle_dev.index: Finished block 10 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:48,469] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:16:48,505] INFO spindle_dev.index: Finished block 11 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:48,506] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:16:48,538] INFO spindle_dev.index: Finished block 12 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:48,538] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:16:48,575] INFO spindle_dev.index: Finished block 13 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:48,575] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:16:48,606] INFO spindle_dev.index: Finished block 14 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:48,607] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:16:48,649] INFO spindle_dev.index: Finished block 15 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:48,650] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:16:48,692] INFO spindle_dev.index: Finished block 16 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:48,693] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:16:48,730] INFO spindle_dev.index: Finished block 17 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:48,731] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:16:48,791] INFO spindle_dev.index: Finished block 18 in 0.06 seconds, found 7 clusters. [2026-01-21 04:16:48,791] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:16:48,842] INFO spindle_dev.index: Finished block 19 in 0.05 seconds, found 4 clusters. [2026-01-21 04:16:48,843] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:16:48,882] INFO spindle_dev.index: Finished block 20 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:48,882] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:16:48,932] INFO spindle_dev.index: Finished block 21 in 0.05 seconds, found 5 clusters. [2026-01-21 04:16:48,932] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:16:48,977] INFO spindle_dev.index: Finished block 22 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:48,977] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:16:49,073] INFO spindle_dev.index: Finished block 23 in 0.10 seconds, found 4 clusters. [2026-01-21 04:16:49,074] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:16:49,124] INFO spindle_dev.index: Finished block 24 in 0.05 seconds, found 5 clusters. [2026-01-21 04:16:49,125] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:16:49,175] INFO spindle_dev.index: Finished block 25 in 0.05 seconds, found 5 clusters. [2026-01-21 04:16:49,175] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:16:49,226] INFO spindle_dev.index: Finished block 26 in 0.05 seconds, found 5 clusters. [2026-01-21 04:16:49,227] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:16:49,272] INFO spindle_dev.index: Finished block 27 in 0.05 seconds, found 4 clusters. [2026-01-21 04:16:49,272] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:16:49,489] INFO spindle_dev.index: Finished block 28 in 0.22 seconds, found 18 clusters. [2026-01-21 04:16:49,490] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:16:49,619] INFO spindle_dev.index: Finished block 29 in 0.13 seconds, found 8 clusters. [2026-01-21 04:16:49,620] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:16:49,620] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:16:49,621] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:16:49,621] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:16:49,621] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:16:49,634] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:16:49,634] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:16:49,634] INFO spindle_dev.index: Processing cluster 2 [2026-01-21 04:16:49,635] INFO spindle_dev.index: Building SPD index with epsilon=8.17919796610604 [2026-01-21 04:16:49,635] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:16:49,656] INFO spindle_dev.index: Cluster 2: 697 SPDs, 30 blocks [2026-01-21 04:16:49,657] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:16:49,682] INFO spindle_dev.index: Finished block 0 in 0.02 seconds, found 1 clusters. [2026-01-21 04:16:49,682] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:16:49,704] INFO spindle_dev.index: Finished block 1 in 0.02 seconds, found 1 clusters. [2026-01-21 04:16:49,705] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:16:49,727] INFO spindle_dev.index: Finished block 2 in 0.02 seconds, found 1 clusters. [2026-01-21 04:16:49,728] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:16:49,755] INFO spindle_dev.index: Finished block 3 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:49,755] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:16:49,788] INFO spindle_dev.index: Finished block 4 in 0.03 seconds, found 3 clusters. [2026-01-21 04:16:49,789] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:16:49,818] INFO spindle_dev.index: Finished block 5 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:49,819] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:16:49,846] INFO spindle_dev.index: Finished block 6 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:49,847] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:16:49,875] INFO spindle_dev.index: Finished block 7 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:49,875] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:16:49,914] INFO spindle_dev.index: Finished block 8 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:49,915] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:16:49,949] INFO spindle_dev.index: Finished block 9 in 0.03 seconds, found 3 clusters. [2026-01-21 04:16:49,949] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:16:49,983] INFO spindle_dev.index: Finished block 10 in 0.03 seconds, found 3 clusters. [2026-01-21 04:16:49,983] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:16:50,014] INFO spindle_dev.index: Finished block 11 in 0.03 seconds, found 2 clusters. [2026-01-21 04:16:50,014] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:16:50,049] INFO spindle_dev.index: Finished block 12 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:50,050] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:16:50,085] INFO spindle_dev.index: Finished block 13 in 0.03 seconds, found 3 clusters. [2026-01-21 04:16:50,085] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:16:50,125] INFO spindle_dev.index: Finished block 14 in 0.04 seconds, found 3 clusters. [2026-01-21 04:16:50,125] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:16:50,160] INFO spindle_dev.index: Finished block 15 in 0.03 seconds, found 3 clusters. [2026-01-21 04:16:50,160] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:16:50,206] INFO spindle_dev.index: Finished block 16 in 0.05 seconds, found 5 clusters. [2026-01-21 04:16:50,206] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:16:50,253] INFO spindle_dev.index: Finished block 17 in 0.05 seconds, found 5 clusters. [2026-01-21 04:16:50,254] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:16:50,300] INFO spindle_dev.index: Finished block 18 in 0.05 seconds, found 5 clusters. [2026-01-21 04:16:50,300] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:16:50,351] INFO spindle_dev.index: Finished block 19 in 0.05 seconds, found 4 clusters. [2026-01-21 04:16:50,352] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:16:50,393] INFO spindle_dev.index: Finished block 20 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:50,394] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:16:50,447] INFO spindle_dev.index: Finished block 21 in 0.05 seconds, found 6 clusters. [2026-01-21 04:16:50,448] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:16:50,495] INFO spindle_dev.index: Finished block 22 in 0.05 seconds, found 5 clusters. [2026-01-21 04:16:50,496] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:16:50,543] INFO spindle_dev.index: Finished block 23 in 0.05 seconds, found 5 clusters. [2026-01-21 04:16:50,544] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:16:50,596] INFO spindle_dev.index: Finished block 24 in 0.05 seconds, found 6 clusters. [2026-01-21 04:16:50,597] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:16:50,646] INFO spindle_dev.index: Finished block 25 in 0.05 seconds, found 4 clusters. [2026-01-21 04:16:50,646] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:16:50,705] INFO spindle_dev.index: Finished block 26 in 0.06 seconds, found 4 clusters. [2026-01-21 04:16:50,706] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:16:50,758] INFO spindle_dev.index: Finished block 27 in 0.05 seconds, found 6 clusters. [2026-01-21 04:16:50,759] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:16:50,820] INFO spindle_dev.index: Finished block 28 in 0.06 seconds, found 6 clusters. [2026-01-21 04:16:50,821] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:16:52,059] INFO spindle_dev.index: Finished block 29 in 1.24 seconds, found 174 clusters. [2026-01-21 04:16:52,060] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:16:52,060] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:16:52,061] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:16:52,061] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:16:52,061] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:16:52,073] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:16:52,073] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:16:52,074] INFO spindle_dev.index: Processing cluster 3 [2026-01-21 04:16:52,074] INFO spindle_dev.index: Building SPD index with epsilon=8.007105696219579 [2026-01-21 04:16:52,074] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:16:52,095] INFO spindle_dev.index: Cluster 3: 297 SPDs, 26 blocks [2026-01-21 04:16:52,096] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:16:52,107] INFO spindle_dev.index: Finished block 0 in 0.01 seconds, found 1 clusters. [2026-01-21 04:16:52,107] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:16:52,117] INFO spindle_dev.index: Finished block 1 in 0.01 seconds, found 1 clusters. [2026-01-21 04:16:52,117] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:16:52,129] INFO spindle_dev.index: Finished block 2 in 0.01 seconds, found 2 clusters. [2026-01-21 04:16:52,130] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:16:52,145] INFO spindle_dev.index: Finished block 3 in 0.02 seconds, found 3 clusters. [2026-01-21 04:16:52,145] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:16:52,158] INFO spindle_dev.index: Finished block 4 in 0.01 seconds, found 2 clusters. [2026-01-21 04:16:52,158] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:16:52,173] INFO spindle_dev.index: Finished block 5 in 0.01 seconds, found 3 clusters. [2026-01-21 04:16:52,174] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:16:52,188] INFO spindle_dev.index: Finished block 6 in 0.01 seconds, found 3 clusters. [2026-01-21 04:16:52,189] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:16:52,202] INFO spindle_dev.index: Finished block 7 in 0.01 seconds, found 2 clusters. [2026-01-21 04:16:52,203] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:16:52,220] INFO spindle_dev.index: Finished block 8 in 0.02 seconds, found 4 clusters. [2026-01-21 04:16:52,220] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:16:52,237] INFO spindle_dev.index: Finished block 9 in 0.02 seconds, found 3 clusters. [2026-01-21 04:16:52,237] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:16:52,252] INFO spindle_dev.index: Finished block 10 in 0.02 seconds, found 3 clusters. [2026-01-21 04:16:52,253] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:16:52,273] INFO spindle_dev.index: Finished block 11 in 0.02 seconds, found 5 clusters. [2026-01-21 04:16:52,273] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:16:52,287] INFO spindle_dev.index: Finished block 12 in 0.01 seconds, found 2 clusters. [2026-01-21 04:16:52,287] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:16:52,301] INFO spindle_dev.index: Finished block 13 in 0.01 seconds, found 2 clusters. [2026-01-21 04:16:52,301] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:16:52,319] INFO spindle_dev.index: Finished block 14 in 0.02 seconds, found 4 clusters. [2026-01-21 04:16:52,319] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:16:52,340] INFO spindle_dev.index: Finished block 15 in 0.02 seconds, found 4 clusters. [2026-01-21 04:16:52,340] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:16:52,358] INFO spindle_dev.index: Finished block 16 in 0.02 seconds, found 3 clusters. [2026-01-21 04:16:52,359] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:16:52,382] INFO spindle_dev.index: Finished block 17 in 0.02 seconds, found 4 clusters. [2026-01-21 04:16:52,383] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:16:52,401] INFO spindle_dev.index: Finished block 18 in 0.02 seconds, found 4 clusters. [2026-01-21 04:16:52,401] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:16:52,430] INFO spindle_dev.index: Finished block 19 in 0.03 seconds, found 3 clusters. [2026-01-21 04:16:52,430] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:16:52,451] INFO spindle_dev.index: Finished block 20 in 0.02 seconds, found 5 clusters. [2026-01-21 04:16:52,452] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:16:52,505] INFO spindle_dev.index: Finished block 21 in 0.05 seconds, found 6 clusters. [2026-01-21 04:16:52,506] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:16:52,547] INFO spindle_dev.index: Finished block 22 in 0.04 seconds, found 4 clusters. [2026-01-21 04:16:52,547] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:16:52,565] INFO spindle_dev.index: Finished block 23 in 0.02 seconds, found 3 clusters. [2026-01-21 04:16:52,565] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:16:52,584] INFO spindle_dev.index: Finished block 24 in 0.02 seconds, found 4 clusters. [2026-01-21 04:16:52,585] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:16:52,662] INFO spindle_dev.index: Finished block 25 in 0.08 seconds, found 7 clusters. [2026-01-21 04:16:52,662] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:16:52,663] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:16:52,663] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:16:52,663] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:16:52,664] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:16:52,668] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:16:52,669] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:16:52,669] INFO spindle_dev.index: Processing cluster 4 [2026-01-21 04:16:52,670] INFO spindle_dev.index: Building SPD index with epsilon=7.326412259271399 [2026-01-21 04:16:52,670] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:16:52,675] INFO spindle_dev.index: Cluster 4: 160 SPDs, 31 blocks [2026-01-21 04:16:52,676] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:16:52,683] INFO spindle_dev.index: Finished block 0 in 0.01 seconds, found 1 clusters. [2026-01-21 04:16:52,683] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:16:52,689] INFO spindle_dev.index: Finished block 1 in 0.01 seconds, found 1 clusters. [2026-01-21 04:16:52,689] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:16:52,694] INFO spindle_dev.index: Finished block 2 in 0.01 seconds, found 1 clusters. [2026-01-21 04:16:52,695] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:16:52,700] INFO spindle_dev.index: Finished block 3 in 0.01 seconds, found 1 clusters. [2026-01-21 04:16:52,700] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:16:52,709] INFO spindle_dev.index: Finished block 4 in 0.01 seconds, found 3 clusters. [2026-01-21 04:16:52,709] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:16:52,716] INFO spindle_dev.index: Finished block 5 in 0.01 seconds, found 2 clusters. [2026-01-21 04:16:52,716] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:16:52,726] INFO spindle_dev.index: Finished block 6 in 0.01 seconds, found 5 clusters. [2026-01-21 04:16:52,727] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:16:52,737] INFO spindle_dev.index: Finished block 7 in 0.01 seconds, found 5 clusters. [2026-01-21 04:16:52,738] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:16:52,748] INFO spindle_dev.index: Finished block 8 in 0.01 seconds, found 5 clusters. [2026-01-21 04:16:52,749] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:16:52,759] INFO spindle_dev.index: Finished block 9 in 0.01 seconds, found 5 clusters. [2026-01-21 04:16:52,760] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:16:52,770] INFO spindle_dev.index: Finished block 10 in 0.01 seconds, found 4 clusters. [2026-01-21 04:16:52,770] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:16:52,780] INFO spindle_dev.index: Finished block 11 in 0.01 seconds, found 4 clusters. [2026-01-21 04:16:52,780] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:16:52,791] INFO spindle_dev.index: Finished block 12 in 0.01 seconds, found 5 clusters. [2026-01-21 04:16:52,791] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:16:52,802] INFO spindle_dev.index: Finished block 13 in 0.01 seconds, found 5 clusters. [2026-01-21 04:16:52,802] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:16:52,812] INFO spindle_dev.index: Finished block 14 in 0.01 seconds, found 4 clusters. [2026-01-21 04:16:52,812] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:16:52,824] INFO spindle_dev.index: Finished block 15 in 0.01 seconds, found 6 clusters. [2026-01-21 04:16:52,824] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:16:52,836] INFO spindle_dev.index: Finished block 16 in 0.01 seconds, found 6 clusters. [2026-01-21 04:16:52,837] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:16:52,848] INFO spindle_dev.index: Finished block 17 in 0.01 seconds, found 5 clusters. [2026-01-21 04:16:52,849] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:16:52,860] INFO spindle_dev.index: Finished block 18 in 0.01 seconds, found 5 clusters. [2026-01-21 04:16:52,861] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:16:52,875] INFO spindle_dev.index: Finished block 19 in 0.01 seconds, found 7 clusters. [2026-01-21 04:16:52,875] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:16:52,895] INFO spindle_dev.index: Finished block 20 in 0.02 seconds, found 9 clusters. [2026-01-21 04:16:52,896] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:16:52,913] INFO spindle_dev.index: Finished block 21 in 0.02 seconds, found 9 clusters. [2026-01-21 04:16:52,914] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:16:52,926] INFO spindle_dev.index: Finished block 22 in 0.01 seconds, found 6 clusters. [2026-01-21 04:16:52,926] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:16:52,940] INFO spindle_dev.index: Finished block 23 in 0.01 seconds, found 7 clusters. [2026-01-21 04:16:52,941] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:16:52,956] INFO spindle_dev.index: Finished block 24 in 0.02 seconds, found 8 clusters. [2026-01-21 04:16:52,957] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:16:52,967] INFO spindle_dev.index: Finished block 25 in 0.01 seconds, found 3 clusters. [2026-01-21 04:16:52,967] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:16:52,979] INFO spindle_dev.index: Finished block 26 in 0.01 seconds, found 5 clusters. [2026-01-21 04:16:52,980] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:16:52,994] INFO spindle_dev.index: Finished block 27 in 0.01 seconds, found 7 clusters. [2026-01-21 04:16:52,994] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:16:53,005] INFO spindle_dev.index: Finished block 28 in 0.01 seconds, found 5 clusters. [2026-01-21 04:16:53,006] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:16:53,040] INFO spindle_dev.index: Finished block 29 in 0.03 seconds, found 13 clusters. [2026-01-21 04:16:53,041] INFO spindle_dev.index: Using epsilon-net clustering for block 30 [2026-01-21 04:16:53,070] INFO spindle_dev.index: Finished block 30 in 0.03 seconds, found 7 clusters. [2026-01-21 04:16:53,070] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:16:53,070] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:16:53,071] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:16:53,071] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:16:53,071] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:16:53,075] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:16:53,075] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances.
In [18]:
search_cfg = search.SearchConfig(max_results=2, debug=False, max_failed_starts=5, max_failed_paths=10)
In [19]:
test_results = test.run_sanity_search(data, dag_dict, config, search_cfg, max_queries=2000, skip_baseline=True)
[2026-01-21 04:16:53,710] INFO spindle_dev.test: Built ground-truth paths for 743 SPDs in cluster 0 across 28 blocks. [2026-01-21 04:16:54,369] INFO spindle_dev.test: Built ground-truth paths for 735 SPDs in cluster 1 across 30 blocks. [2026-01-21 04:16:54,972] INFO spindle_dev.test: Built ground-truth paths for 697 SPDs in cluster 2 across 30 blocks. [2026-01-21 04:16:55,059] INFO spindle_dev.test: Built ground-truth paths for 297 SPDs in cluster 3 across 26 blocks. [2026-01-21 04:16:55,093] INFO spindle_dev.test: Built ground-truth paths for 160 SPDs in cluster 4 across 31 blocks. [2026-01-21 04:16:55,094] INFO spindle_dev.test: Created ground-truth paths for all clusters. [2026-01-21 04:16:55,094] INFO spindle_dev.test: Search config: SearchConfig(max_results=2, max_failed_starts=5, max_failed_paths=10, total_paths_limit=1000, deterministic=DeterministicConfig(seed=0), debug=False) [2026-01-21 04:16:55,095] INFO spindle_dev.test: Running sanity search with 2000 queries. 100%|██████████| 2000/2000 [00:22<00:00, 89.14it/s] [2026-01-21 04:17:17,534] INFO spindle_dev.test: Sanity search: 2000 queries, 2000 exact path matches, 2000 leaf matches, mean search_time=0.0106s
In [20]:
fig = plotting.visualize_block_dag_sankey_scaled(dag_dict[0].nodes, height=800, thickness=20.0)
fig
In [21]:
import pandas as pd
def get_dag_stats(dag_dict):
dag_stats = []
for cluster_id, cluster_dag in dag_dict.items():
block_to_node = cluster_dag.block_to_node_indices
for node in cluster_dag.nodes:
num_spds = len(node.metadata.members)
block_size = node.metadata.mean.shape[0]
block_id = node.block_index
radius = node.metadata.radius
#print(f"Cluster_id {cluster_id} Node id {node.global_node_id} Block {block_id}: {num_spds} spds")
dag_stats.append({
"cluster_id": cluster_id,
"node_id": node.global_node_id,
"block_id": block_id,
"block_cluster_id": node.block_cluster_id,
"num_spds": num_spds,
"radius": radius,
"block_size": block_size
})
dag_stats_df = pd.DataFrame(dag_stats)
return dag_stats_df
In [22]:
dag_stats_df = get_dag_stats(dag_dict)
In [23]:
score_list = []
for cluster_id in set(data.labels):
cluster_dag = dag_dict[cluster_id]
cluster_score = index.score_nodes_from_a_cluster(data, cluster_id, dag_dict, take_representative_mean=True)
score_list.extend(cluster_score)
100%|██████████| 117/117 [00:00<00:00, 160.54it/s] 100%|██████████| 116/116 [00:00<00:00, 297.55it/s] 100%|██████████| 277/277 [00:04<00:00, 69.15it/s] 100%|██████████| 87/87 [00:00<00:00, 297.63it/s] 100%|██████████| 159/159 [00:00<00:00, 501.59it/s]
In [24]:
node_scores = []
for node_score in score_list:
node_scores.append({
"cluster_id": node_score["cluster_id"],
"block_id": node_score["block_id"],
"node_id": node_score["node_id"],
"node_score": node_score["node_score"],
"num_spds": node_score["num_spds"],
"block_size": node_score["block_size"],
"E": node_score["E"],
"Q": node_score["Q"],
"S": node_score["S"],
"num_modules": len(node_score["modules"])
})
node_score_df = pd.DataFrame(node_scores)
In [25]:
node_score_df.sort_values(by='node_score', ascending=False).head(20)
Out[25]:
| cluster_id | block_id | node_id | node_score | num_spds | block_size | E | Q | S | num_modules | |
|---|---|---|---|---|---|---|---|---|---|---|
| 456 | 2 | 29 | 223 | 0.279565 | 1 | 49 | 2.433631 | 0.212817 | 0.460218 | 3 |
| 345 | 2 | 29 | 112 | 0.277771 | 1 | 49 | 2.209249 | 0.211656 | 0.405969 | 5 |
| 387 | 2 | 29 | 154 | 0.226938 | 2 | 49 | 3.014656 | 0.161094 | 0.532710 | 3 |
| 375 | 2 | 29 | 142 | 0.217738 | 1 | 49 | 2.739329 | 0.163122 | 0.512724 | 2 |
| 349 | 2 | 29 | 116 | 0.201381 | 1 | 49 | 2.617111 | 0.155838 | 0.506235 | 2 |
| 185 | 1 | 23 | 68 | 0.198142 | 6 | 29 | 1.197445 | 0.326108 | 0.492592 | 6 |
| 470 | 2 | 29 | 237 | 0.196448 | 1 | 49 | 2.102675 | 0.163682 | 0.429215 | 4 |
| 481 | 2 | 29 | 248 | 0.196273 | 1 | 49 | 1.967417 | 0.168113 | 0.406583 | 3 |
| 594 | 3 | 25 | 84 | 0.195628 | 3 | 44 | 1.834050 | 0.182488 | 0.415502 | 3 |
| 492 | 2 | 29 | 259 | 0.191804 | 1 | 49 | 3.073824 | 0.138394 | 0.549123 | 3 |
| 441 | 2 | 29 | 208 | 0.189483 | 1 | 49 | 2.144042 | 0.161692 | 0.453430 | 3 |
| 482 | 2 | 29 | 249 | 0.187372 | 1 | 49 | 2.331694 | 0.152117 | 0.471734 | 3 |
| 575 | 3 | 21 | 65 | 0.187352 | 4 | 33 | 0.690722 | 0.393953 | 0.311491 | 11 |
| 663 | 4 | 17 | 66 | 0.183470 | 23 | 12 | 0.647413 | 0.475616 | 0.404165 | 4 |
| 369 | 2 | 29 | 136 | 0.177440 | 1 | 49 | 2.858447 | 0.125056 | 0.503623 | 3 |
| 431 | 2 | 29 | 198 | 0.176948 | 1 | 49 | 2.352287 | 0.141815 | 0.469567 | 3 |
| 443 | 2 | 29 | 210 | 0.173004 | 1 | 49 | 2.400901 | 0.144759 | 0.502227 | 3 |
| 361 | 2 | 29 | 128 | 0.172351 | 1 | 49 | 1.939020 | 0.151129 | 0.411861 | 3 |
| 67 | 0 | 23 | 67 | 0.172152 | 33 | 18 | 1.162531 | 0.299296 | 0.505228 | 4 |
| 484 | 2 | 29 | 251 | 0.170435 | 1 | 49 | 1.908461 | 0.156855 | 0.430656 | 5 |
In [36]:
id = 456
scores_846, fig_846 = plotting.plot_module_heatmap_plus_spatial(
id,
score_list,
node_score_df,
dag_stats_df,
dag_dict,
data,
adata
)
Creating combined figure for cluster 2, node 223, block 29
In [37]:
fig_846.savefig(f"{result_dir}/{id}_module_0_1_enrichment.png", dpi=300)
In [38]:
from spindle_dev import go_score
In [39]:
out_846 = go_score.enrich_modules_with_gseapy(score_list[id]['modules'])
Module 0: enrichment completed with 476 terms Module 1: enrichment completed with 733 terms Module 2: enrichment completed with 229 terms
In [30]:
for module_id, module_genes, res in out_846:
if module_id == 1:
print(module_genes)
fig, axes = go_score.plot_module_enrichment_libraries(
module_id=module_id,
module_genes=module_genes,
res=res,
top_n=7,
ncols=2,
bar_color="#D41515EE", # pick your color
tick_fontsize=10,
label_fontsize=10,
title_fontsize=8,
compact_height_per_term=0.2,
min_fig_h=6,
save_dir=f"{result_dir}/go_enrichment/enrichment_{id}"
)
if fig is not None:
plt.show()
['CD163', 'IRF8', 'LYVE1', 'MNDA', 'MS4A4A'] ['GO_Biological_Process_2023', 'KEGG_2021_Human', 'MSigDB_Hallmark_2020', 'Reactome_2022']
In [40]:
for module_id, module_genes, res in out_846:
if module_id == 1:
print(module_genes)
fig, axes = go_score.plot_module_enrichment_libraries_2(
module_id=module_id,
module_genes=module_genes,
res=res,
top_n=7,
ncols=2,
bar_color="#D41515EE", # pick your color
tick_fontsize=10,
label_fontsize=10,
title_fontsize=8,
compact_height_per_term=0.2,
min_fig_h=8,
plot_kind="dot",
size_range=(20, 200),
use_constrained_layout=False,
save_dir=f"{result_dir}/go_enrichment/enrichment_{id}"
)
if fig is not None:
plt.show()
['ACTG2', 'ADAM28', 'AMY2A', 'BCL2L11', 'CFB', 'DST', 'EGFR', 'EPCAM', 'ERBB2', 'FAS', 'GPC3', 'MALL', 'MCF2L', 'MDM2', 'MET', 'MYLK', 'NTN4', 'PCNA', 'S100A1', 'SMYD2', 'VWA5A']
/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/go_score.py:519: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
In [31]:
# Create a dictionary of tile_id to block_id
# Clusters do not share tile ids across them
from collections import defaultdict
tile_to_block_dict = {}
cluster_to_tile = defaultdict(set)
for cluster_id, cluster_dag in dag_dict.items():
block_to_node = cluster_dag.block_to_node_indices
for node in cluster_dag.nodes:
member_tile_ids = [tid for tid,_ in node.metadata.members]
for tile_id in member_tile_ids:
if not tile_id in tile_to_block_dict:
tile_to_block_dict[tile_id] = {}
tile_to_block_dict[tile_id][node.block_index] = node.block_cluster_id
cluster_to_tile[cluster_id].add(tile_id)
In [32]:
def get_spot_score_dict(block_id, cluster_id, tile_to_block_dict, grid=False):
spot_score = {}
tiles_of_cluster = cluster_to_tile[cluster_id]
for tid in tiles_of_cluster:
if tid not in tile_to_block_dict:
continue
c = tile_to_block_dict[tid][block_id]
t = data.metadata['tiles'][tid]
for id in t.idx:
spot_score[id] = c
fig, ax = plotting.plot_spot_module_scores(spot_score, tiles, adata, color='discrete', grid=grid)
return fig, ax
In [33]:
fig, ax = get_spot_score_dict(5, 4, tile_to_block_dict, grid=False)
In [34]:
sc.pl.spatial(adata,color=['AQP3', 'AHNAK2'], spot_size=20, cmap='Reds')
/tmp/ipykernel_4150978/909464612.py:1: FutureWarning: Use `squidpy.pl.spatial_scatter` instead.
--------------------------------------------------------------------------- KeyError Traceback (most recent call last) Cell In[34], line 1 ----> 1 sc.pl.spatial(adata,color=['AQP3', 'AHNAK2'], spot_size=20, cmap='Reds') File /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/typing_extensions.py:3004, in deprecated.__call__.<locals>.wrapper(*args, **kwargs) 3001 @functools.wraps(arg) 3002 def wrapper(*args, **kwargs): 3003 warnings.warn(msg, category=category, stacklevel=stacklevel + 1) -> 3004 return arg(*args, **kwargs) File /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/scanpy/plotting/_tools/scatterplots.py:1018, in spatial(adata, basis, img, img_key, library_id, crop_coord, alpha_img, bw, size, scale_factor, spot_size, na_color, show, return_fig, save, **kwargs) 1015 cmap_img = "gray" if bw else None 1016 circle_radius = size * scale_factor * spot_size * 0.5 -> 1018 axs = embedding( 1019 adata, 1020 basis=basis, 1021 scale_factor=scale_factor, 1022 size=circle_radius, 1023 na_color=na_color, 1024 show=False, 1025 save=False, 1026 **kwargs, 1027 ) 1028 if not isinstance(axs, list): 1029 axs = [axs] File /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/scanpy/plotting/_tools/scatterplots.py:279, in embedding(adata, basis, color, mask_obs, gene_symbols, use_raw, sort_order, edges, edges_width, edges_color, neighbors_key, arrows, arrows_kwds, groups, components, dimensions, layer, projection, scale_factor, color_map, cmap, palette, na_color, na_in_legend, size, frameon, legend_fontsize, legend_fontweight, legend_loc, legend_fontoutline, colorbar_loc, vmax, vmin, vcenter, norm, add_outline, outline_width, outline_color, ncols, hspace, wspace, title, show, save, ax, return_fig, marker, **kwargs) 277 for count, (value_to_plot, dims) in enumerate(zip(color, dimensions, strict=True)): 278 kwargs_scatter = kwargs.copy() # is potentially mutated for each plot --> 279 color_source_vector = _get_color_source_vector( 280 adata, 281 value_to_plot, 282 layer=layer, 283 mask_obs=mask_obs, 284 use_raw=use_raw, 285 gene_symbols=gene_symbols, 286 groups=groups, 287 ) 288 color_vector, color_type = _color_vector( 289 adata, 290 value_to_plot, (...) 293 na_color=na_color, 294 ) 296 # Order points File /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/scanpy/plotting/_tools/scatterplots.py:1200, in _get_color_source_vector(adata, value_to_plot, mask_obs, use_raw, gene_symbols, layer, groups) 1198 values = adata.raw.obs_vector(value_to_plot) 1199 else: -> 1200 values = adata.obs_vector(value_to_plot, layer=layer) 1201 if mask_obs is not None: 1202 values = values.copy() File /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/anndata/_core/anndata.py:1297, in AnnData.obs_vector(self, k, layer) 1291 warnings.warn( 1292 "In a future version of AnnData, access to `.X` by passing" 1293 " `layer='X'` will be removed. Instead pass `layer=None`.", 1294 FutureWarning, 1295 ) 1296 layer = None -> 1297 return get_vector(self, k, "obs", "var", layer=layer) File /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/anndata/_core/index.py:245, in get_vector(adata, k, coldim, idxdim, layer) 243 elif (in_col + in_idx) == 0: 244 msg = f"Could not find key {k} in .{idxdim}_names or .{coldim}.columns." --> 245 raise KeyError(msg) 246 elif in_col: 247 return getattr(adata, coldim)[k].values KeyError: 'Could not find key AHNAK2 in .var_names or .obs.columns.'
In [ ]: