from pathlib import Path
import os
PROJECT_ROOT = Path('/Users/weizexu/Projects/U-Chrom')
os.chdir(PROJECT_ROOT)
print('cwd:', Path.cwd())

cwd: /Users/weizexu/Projects/U-Chrom

from pathlib import Path
import json
import os
os.environ.setdefault('MPLBACKEND', 'Agg')
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('Agg', force=True)
import matplotlib.pyplot as plt
from uchrom import ChromData
from uchrom.auto_discovery import DiscoveryIdea, review_idea_against_schema

IDEA = DiscoveryIdea.from_dict({'idea_title': 'Aldoc expression tracks lamina-associated chromatin signal', 'biological_hypothesis': 'Cells with higher Aldoc RNA expression exhibit increased LaminB1-associated chromatin signal, linking expression state to lamina-proximal chromatin organization.', 'computable_parameter': 'Spearman rho between per-cell Aldoc expression and per-cell mean tracks.LaminB1 over all spots.', 'analysis_plan': 'Extract linked_adata.X values for linked_adata.var.Aldoc and align them to cells. Group spot rows by spots.cell_id and compute the mean tracks.LaminB1 for each cell. Use cells.cell_type only for stratified reporting and minimum-count checks, while the primary measurement is the single across-cell Spearman correlation between Aldoc expression and mean LaminB1 signal. Assess significance with the reported p-value and a fixed-seed permutation of Aldoc expression across cells.', 'modalities': ['if_tracks', 'cell_metadata', 'rna_expression'], 'idea_markdown': '### Rationale\nAldoc is available in the linked RNA matrix and may mark cell-state differences that are reflected in nuclear peripheral organization.\n\n### Data used\nUse linked Aldoc expression, cell type labels, spot-to-cell assignments, and the LaminB1 IF track measured on chromatin spots.\n\n### Analysis sketch\nFor each cell, compute the mean LaminB1 signal over all chromatin spots assigned to that cell. Correlate this per-cell LaminB1 chromatin-associated signal with Aldoc expression.\n\n### Expected result\nIf Aldoc-high cells have more lamina-proximal or lamina-associated chromatin, Aldoc expression should increase with mean LaminB1 signal.\n\n### Validation checks\nCheck all required fields, verify sufficient cells and spots per cell, require finite means, report a Spearman p-value, rerun deterministically, and use shuffled Aldoc labels as a negative control.', 'cell_types': ['Granule', 'Bergmann', 'Purkinje'], 'required_fields': ['spots.cell_id', 'tracks.LaminB1', 'cells.cell_type', 'linked_adata.X', 'linked_adata.var.Aldoc'], 'validation_checks': ['required_fields_exist', 'minimum_cell_count_n>=9_and_each_listed_cell_type_n>=3', 'minimum_spot_or_trace_count_per_cell_for_LaminB1_mean', 'finite_numeric_output', 'statistical_hypothesis_test_spearman_with_p_value', 'runtime_under_budget', 'deterministic_rerun', 'negative_control_or_permutation_by_shuffling_Aldoc_expression_across_cells'], 'expected_direction': 'Positive correlation: higher Aldoc expression should correspond to higher mean chromatin-associated LaminB1 signal.', 'complexity': 2, 'idea_id': 'aldoc-expression-tracks-lamina-associated-chroma-4c06d0d43e', 'metadata': {}})
H5CD_PATH = 'tmp/takei_auto_discovery_doc/takei_doc_auto_subset.h5cd'
RUN_OUTPUT_DIR = Path('tmp/takei_auto_discovery_doc/run_pantheon_20_ideas_verified_agg')
RUN_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
cdata = ChromData.read(H5CD_PATH) if H5CD_PATH else None
schema = cdata.discovery_schema if cdata is not None else None
adata = cdata.linked_adata if cdata is not None else None
print(IDEA.idea_id)
if cdata is not None:
    print(cdata)
    print(cdata.describe_for_agent(max_items=20))

aldoc-expression-tracks-lamina-associated-chroma-4c06d0d43e
ChromData: n_spots=56036, n_traces=213, n_cells=9
  spots:   ['chrom', 'start', 'end', 'trace_id', 'cell_id', 'name']
  cells:   ['leiden', 'cell_type', 'x_centroid', 'y_centroid', 'z_centroid', 'nuc_volume_um3', 'doublet', 'batch', 'n_transcripts', 'n_genes_by_counts'] (9 cells)
  cellm:   {'umap': (9, 2)}
  tracks:  ['CPSF6', 'ATRX', 'H4K8ac', 'HDAC2', 'H3K9ac', 'H3K9me3', 'H3K9me2', 'RNAPIISer2-P', 'H3', 'H3K36me2', 'UBTF', 'LaminB1', 'RNAPIISer5-P', 'RYBP', 'HP1beta', 'RING1B', 'H2A.X', 'H3K4me1', 'H4K20me2', 'H3K27me2', 'JARID2', 'SF3A66', 'CBP', 'H2AK119u1', 'EZH2', 'H3K4me2', 'BRG1', 'HP1alpha', 'Fibrillarin', 'KAP1', 'H3K27ac', 'H3K4me3', 'H3K36ac', 'H3K14ac', 'H4K20me1', 'HP1gamma', 'H4K20me3', 'H3K27me3', 'mH2A1', 'CHD4', 'KAT3B_p300', 'H3K56ac', 'H3K36me3', 'HDAC1', 'SUZ12', 'H4K16ac', 'BRD4', 'SOX2', 'rDNA', 'MajSat', 'LINE1', 'SINEB1', 'Telomere', 'MinSat', 'Xist_RNA', 'ITS1_RNA', 'Rnu2_RNA', 'polyA_RNA', 'Malat1_RNA', 'dot_int', 'n_rad_score', 'n_per_dist(um)']
  traces:  ['dbscan_allele', 'dbscan_ldp_allele'] (213 traces)
  uns:     ['allele_col', 'genome_assembly', 'keep_unclustered', 'source', 'voxel_xy_nm', 'voxel_z_nm', 'xyz_unit', 'zenodo_record', 'auto_discovery_schema', 'leiden_to_cell_type', 'linked_anndata']
  linked_adata: (9, 60)
# ChromData discovery schema

dataset: takei2025_doc_subset_pantheon_20
genome: mm10
xyz_unit: um
shape: 56036 spots, 213 traces, 9 cells

modalities:
- cell_metadata: present; operations: cell_type_stratification, embedding_visualization
- chromatin_tracing: present; operations: chromosome_subset, cell_subset, trace_subset, pairwise_3d_distance, intra_chromatin_distance, inter_chromatin_distance
- if_tracks: present; operations: marker_high_low_bin_selection, marker_stratified_distance, per_cell_marker_summary, per_cell_type_marker_summary
- rna_expression: present; operations: gene_expression_lookup, expression_stratification, gene_marker_correlation, chromatin_expression_association

chroms: 20 [chr1, chr10, chr11, chr12, chr13, chr14, chr15, chr16, chr17, chr18, chr19, chr2, chr3, chr4, chr5, chr6, chr7, chr8, chr9, chrX]
cell_types: 3 [Bergmann=3, Granule=3, Purkinje=3]
tracks: 62 [CPSF6, ATRX, H4K8ac, HDAC2, H3K9ac, H3K9me3, H3K9me2, RNAPIISer2-P, H3, H3K36me2, UBTF, LaminB1, RNAPIISer5-P, RYBP, HP1beta, RING1B, H2A.X, H3K4me1, H4K20me2, H3K27me2 ...]
linked_adata: shape=[9, 60], X=csr_matrix
genes: 60 [Aldoc, Calb1, Cdh22, Drd3, Eomes, Ephb2, Foxj1, Gabra6, Gpr176, Grm1, Hspb1, Mrc1, Nefh, Npas3, Nptn, Olig1, Pcp2, Pcp4, Plcb3, Plcb4 ...]

known_missing:
- cellm['if_mean'] per-cell IF mean matrix
- raw RNA seqFISH spot geometry as a first-class ChromData component
- scRNA reference matrix for external expression comparison
- gene annotation cache for gene-neighborhood analyses

verification_required:
- required_fields_exist
- minimum_cell_count
- minimum_spot_or_trace_count
- finite_numeric_output
- statistical_hypothesis_test
- runtime_under_budget
- deterministic_rerun
- negative_control_or_permutation
- redundancy_against_existing_parameters

review = review_idea_against_schema(IDEA, schema) if schema is not None else None
print(None if review is None else review.to_dict())
assert review is None or review.accepted, review.to_dict()

{'accepted': True, 'errors': [], 'warnings': ['multi-modal idea should include a cell_id_alignment validation check'], 'missing_fields': []}

# Lightweight data inspection: required columns, alignment assumptions, finite coverage.
import numpy as np
import pandas as pd

print('spots shape:', cdata.spots.shape)
print('cells shape:', cdata.cells.shape)
print('tracks shape:', cdata.tracks.shape)
print('linked_adata shape:', adata.shape)
print('cells index head:', list(map(str, cdata.cells.index[:5])))
print('adata obs_names head:', list(map(str, adata.obs_names[:5])))
print('cell_id aligned:', list(map(str, cdata.cells.index)) == list(map(str, adata.obs_names)))
print('cell_type counts:', cdata.cells['cell_type'].value_counts().to_dict())

lamin = pd.to_numeric(cdata.tracks['LaminB1'], errors='coerce')
spot_counts = cdata.spots.groupby('cell_id', observed=True).size().rename('n_spots')
finite_counts = lamin.groupby(cdata.spots['cell_id'], observed=True).apply(lambda s: int(np.isfinite(s).sum())).rename('finite_LaminB1')
preview = pd.concat([spot_counts, finite_counts], axis=1).join(cdata.cells[['cell_type']], how='left')
print('per-cell spot/LaminB1 coverage:')
display(preview)

aldoc_idx = list(adata.var_names).index('Aldoc')
aldoc_expr = np.asarray(adata.X[:, aldoc_idx].toarray()).ravel() if hasattr(adata.X, 'toarray') else np.asarray(adata.X[:, aldoc_idx]).ravel()
expr_preview = pd.DataFrame({'cell_id': list(map(str, adata.obs_names)), 'Aldoc_expression': aldoc_expr})
print('Aldoc expression preview:')
display(expr_preview)

spots shape: (56036, 6)
cells shape: (9, 10)
tracks shape: (56036, 62)
linked_adata shape: (9, 60)
cells index head: ['1_0_42', '1_0_47', '1_0_69', '1_0_34', '1_0_61']
adata obs_names head: ['1_0_42', '1_0_47', '1_0_69', '1_0_34', '1_0_61']
cell_id aligned: True
cell_type counts: {'Granule': 3, 'Bergmann': 3, 'Purkinje': 3}
per-cell spot/LaminB1 coverage:
         n_spots  finite_LaminB1 cell_type
cell_id                                   
1_0_116    11659           11659  Purkinje
1_0_26      4225            4225  Purkinje
1_0_34      3932            3932  Bergmann
1_0_37      5238            5238  Purkinje
1_0_42      4183            4183   Granule
1_0_47      4682            4682   Granule
1_0_61     11283           11283  Bergmann
1_0_63      7614            7614  Bergmann
1_0_69      3220            3220   Granule
Aldoc expression preview:
   cell_id  Aldoc_expression
0   1_0_42               1.0
1   1_0_47               1.0
2   1_0_69               0.0
3   1_0_34              27.0
4   1_0_61              75.0
5   1_0_63              72.0
6   1_0_26              50.0
7   1_0_37              79.0
8  1_0_116             327.0

# Main compact exploration: Aldoc expression vs per-cell mean LaminB1 spot signal.
import os
os.environ.setdefault('MPLBACKEND', 'Agg')
import matplotlib
matplotlib.use('Agg', force=True)
import matplotlib.pyplot as plt
from pathlib import Path
import json
import numpy as np
import pandas as pd
from scipy.stats import spearmanr
from IPython.display import Image, display

PROJECT_ROOT = Path('/Users/weizexu/Projects/U-Chrom')
REL_FIG_PATH = Path('tmp/takei_auto_discovery_doc/run_pantheon_20_ideas_verified_agg/aldoc-expression-tracks-lamina-associated-chroma-4c06d0d43e_statistical_summary.png')
REL_RESULT_PATH = Path('tmp/takei_auto_discovery_doc/run_pantheon_20_ideas_verified_agg/aldoc-expression-tracks-lamina-associated-chroma-4c06d0d43e_result.csv')
FIG_PATH = PROJECT_ROOT / REL_FIG_PATH
RESULT_PATH = PROJECT_ROOT / REL_RESULT_PATH
FIG_PATH.parent.mkdir(parents=True, exist_ok=True)
RESULT_PATH.parent.mkdir(parents=True, exist_ok=True)

# Align cell metadata and linked Aldoc expression by cell id.
cell_ids = pd.Index(map(str, cdata.cells.index), name='cell_id')
adata_obs = pd.Index(map(str, adata.obs_names), name='cell_id')
if not cell_ids.equals(adata_obs):
    raise ValueError('cdata.cells.index and linked_adata.obs_names are not aligned; refusing implicit merge')
if 'Aldoc' not in adata.var_names:
    raise ValueError('Aldoc not found in linked_adata.var_names')

aldoc_idx = list(adata.var_names).index('Aldoc')
aldoc_expr = np.asarray(adata.X[:, aldoc_idx].toarray()).ravel() if hasattr(adata.X, 'toarray') else np.asarray(adata.X[:, aldoc_idx]).ravel()

# Per-cell LaminB1 summary over all assigned spots.
spot_cell = cdata.spots['cell_id'].astype(str)
lamin = pd.to_numeric(cdata.tracks['LaminB1'], errors='coerce')
per_cell_lamin = lamin.groupby(spot_cell, observed=True).agg(['mean', 'count']).rename(columns={'mean': 'mean_LaminB1', 'count': 'n_finite_LaminB1_spots'})
spot_counts = cdata.spots.groupby(spot_cell, observed=True).size().rename('n_spots')

cell_df = pd.DataFrame({
    'cell_id': cell_ids.astype(str),
    'cell_type': cdata.cells['cell_type'].astype(str).to_numpy(),
    'Aldoc_expression': pd.to_numeric(aldoc_expr, errors='coerce'),
}).set_index('cell_id')
cell_df = cell_df.join(per_cell_lamin, how='left').join(spot_counts, how='left')
cell_df['n_spots'] = cell_df['n_spots'].fillna(0).astype(int)
cell_df['n_finite_LaminB1_spots'] = cell_df['n_finite_LaminB1_spots'].fillna(0).astype(int)
cell_df['finite_for_test'] = np.isfinite(cell_df['Aldoc_expression']) & np.isfinite(cell_df['mean_LaminB1'])

valid = cell_df.loc[cell_df['finite_for_test']].copy()
n = len(valid)
min_spots_per_cell = int(valid['n_finite_LaminB1_spots'].min()) if n else 0

null_hypothesis = 'Across cells, Aldoc expression is not positively associated with per-cell mean LaminB1 chromatin spot signal; Spearman rho is <= 0 or no greater than label-shuffle expectation.'
alternative_hypothesis = 'Across cells, higher Aldoc expression is associated with higher per-cell mean LaminB1 chromatin spot signal; Spearman rho is > 0.'
test_method = 'Spearman correlation with fixed-seed one-sided permutation test (1000 Aldoc-label shuffles)'

rng = np.random.default_rng(4060)
n_perm = 1000
if n >= 3 and valid['Aldoc_expression'].nunique() >= 2 and valid['mean_LaminB1'].nunique() >= 2:
    observed_rho = float(spearmanr(valid['Aldoc_expression'], valid['mean_LaminB1']).statistic)
    perm_rhos = np.empty(n_perm, dtype=float)
    y = valid['mean_LaminB1'].to_numpy(dtype=float)
    x = valid['Aldoc_expression'].to_numpy(dtype=float)
    for i in range(n_perm):
        perm_rhos[i] = float(spearmanr(rng.permutation(x), y).statistic)
    # One-sided positive association p-value; add-one smoothing for bounded randomization inference.
    p_value = float((np.sum(perm_rhos >= observed_rho) + 1) / (n_perm + 1))
    effect_size = observed_rho
    hypothesis_test_status = 'pass'
    test_note = 'Permutation p-value tests positive Spearman association by shuffling Aldoc labels across cells.'
else:
    observed_rho = float('nan')
    perm_rhos = np.array([], dtype=float)
    p_value = float('nan')
    effect_size = float('nan')
    hypothesis_test_status = 'insufficient_data'
    test_note = 'Insufficient finite cells or variability for Spearman/permutation test.'

# Include the test result columns on every cell-level row so the CSV is self-contained.
result_table = valid.reset_index().copy()
result_table['observed_statistic'] = observed_rho
result_table['effect_size'] = effect_size
result_table['p_value'] = p_value
result_table['test_method'] = test_method
result_table['hypothesis_test_status'] = hypothesis_test_status
result_table['expected_direction'] = 'positive'
result_table.to_csv(RESULT_PATH, index=False)

# Stratified descriptive reporting, not used as the primary hypothesis test.
stratified = valid.groupby('cell_type', observed=True).agg(
    n_cells=('Aldoc_expression', 'size'),
    mean_Aldoc=('Aldoc_expression', 'mean'),
    mean_LaminB1=('mean_LaminB1', 'mean'),
    min_spots=('n_finite_LaminB1_spots', 'min'),
).reset_index()

analysis_summary = {
    'idea_id': IDEA.idea_id,
    'parameter_name': 'Spearman rho between per-cell Aldoc expression and mean tracks.LaminB1',
    'parameter_value': observed_rho,
    'observed_statistic': observed_rho,
    'effect_size': effect_size,
    'p_value': p_value,
    'test_method': test_method,
    'null_hypothesis': null_hypothesis,
    'alternative_hypothesis': alternative_hypothesis,
    'hypothesis_test_status': hypothesis_test_status,
    'n_selected_cells': int(n),
    'n_rows': int(len(result_table)),
    'n_permutations': int(n_perm if hypothesis_test_status == 'pass' else 0),
    'min_finite_LaminB1_spots_per_cell': min_spots_per_cell,
    'cell_type_counts': valid['cell_type'].value_counts().sort_index().to_dict(),
    'result_path': str(REL_RESULT_PATH),
    'figure_path': str(REL_FIG_PATH),
    'notes': [test_note, 'Primary association pools all 9 cells; cell type is reported descriptively only because n=3 per type.'],
}

# Statistical figure: data relationship and observed statistic vs permutation null.
fig, axes = plt.subplots(1, 2, figsize=(11, 4.6), facecolor='white')
colors = {'Granule': '#1f77b4', 'Bergmann': '#ff7f0e', 'Purkinje': '#2ca02c'}
for cell_type, sub in valid.groupby('cell_type', observed=True):
    axes[0].scatter(
        sub['Aldoc_expression'], sub['mean_LaminB1'],
        s=70, alpha=0.9, edgecolor='black', linewidth=0.6,
        label=f'{cell_type} (n={len(sub)})', color=colors.get(cell_type, 'gray')
    )
# Simple least-squares guide line in data coordinates, not used for inference.
if n >= 2:
    coeff = np.polyfit(valid['Aldoc_expression'].to_numpy(float), valid['mean_LaminB1'].to_numpy(float), deg=1)
    xs = np.linspace(valid['Aldoc_expression'].min(), valid['Aldoc_expression'].max(), 100)
    axes[0].plot(xs, coeff[0] * xs + coeff[1], color='black', linestyle='--', linewidth=1.0, label='linear guide')
axes[0].set_xlabel('Aldoc expression (linked RNA counts)')
axes[0].set_ylabel('Mean LaminB1 signal per chromatin spot (a.u.)')
axes[0].set_title('Cell-level Aldoc vs LaminB1')
axes[0].legend(frameon=False, fontsize=8)
axes[0].grid(True, alpha=0.25)

if len(perm_rhos):
    axes[1].hist(perm_rhos, bins=25, color='#c7c7c7', edgecolor='white', label='Aldoc-label shuffle null')
    axes[1].axvline(observed_rho, color='#d62728', linewidth=2.2, label=f'observed rho={observed_rho:.3f}')
    axes[1].axvline(np.nanmean(perm_rhos), color='black', linestyle=':', linewidth=1.2, label=f'null mean={np.nanmean(perm_rhos):.3f}')
else:
    axes[1].text(0.5, 0.5, 'Insufficient data for permutation null', ha='center', va='center')
axes[1].set_xlabel('Spearman rho under shuffled Aldoc labels')
axes[1].set_ylabel('Permutation count')
axes[1].set_title('Hypothesis-test evidence')
axes[1].legend(frameon=False, fontsize=8)
axes[1].grid(True, alpha=0.25)

fig.suptitle('Aldoc expression and lamina-associated chromatin signal', fontsize=13)
fig.text(0.5, 0.01, f'{test_method}; one-sided p={p_value:.4f}; effect size=rho={effect_size:.3f}; n={n} cells', ha='center', fontsize=9)
fig.tight_layout(rect=(0, 0.04, 1, 0.94))
fig.savefig(FIG_PATH, dpi=180, bbox_inches='tight')
plt.close(fig)
display(Image(filename=str(FIG_PATH)))

display(result_table)
print(json.dumps(analysis_summary, indent=2))

<IPython.core.display.Image object>
   cell_id cell_type  ...  hypothesis_test_status  expected_direction
0   1_0_42   Granule  ...                    pass            positive
1   1_0_47   Granule  ...                    pass            positive
2   1_0_69   Granule  ...                    pass            positive
3   1_0_34  Bergmann  ...                    pass            positive
4   1_0_61  Bergmann  ...                    pass            positive
5   1_0_63  Bergmann  ...                    pass            positive
6   1_0_26  Purkinje  ...                    pass            positive
7   1_0_37  Purkinje  ...                    pass            positive
8  1_0_116  Purkinje  ...                    pass            positive

[9 rows x 13 columns]
{
  "idea_id": "aldoc-expression-tracks-lamina-associated-chroma-4c06d0d43e",
  "parameter_name": "Spearman rho between per-cell Aldoc expression and mean tracks.LaminB1",
  "parameter_value": 0.5355695416036628,
  "observed_statistic": 0.5355695416036628,
  "effect_size": 0.5355695416036628,
  "p_value": 0.07092907092907093,
  "test_method": "Spearman correlation with fixed-seed one-sided permutation test (1000 Aldoc-label shuffles)",
  "null_hypothesis": "Across cells, Aldoc expression is not positively associated with per-cell mean LaminB1 chromatin spot signal; Spearman rho is <= 0 or no greater than label-shuffle expectation.",
  "alternative_hypothesis": "Across cells, higher Aldoc expression is associated with higher per-cell mean LaminB1 chromatin spot signal; Spearman rho is > 0.",
  "hypothesis_test_status": "pass",
  "n_selected_cells": 9,
  "n_rows": 9,
  "n_permutations": 1000,
  "min_finite_LaminB1_spots_per_cell": 3220,
  "cell_type_counts": {
    "Bergmann": 3,
    "Granule": 3,
    "Purkinje": 3
  },
  "result_path": "tmp/takei_auto_discovery_doc/run_pantheon_20_ideas_verified_agg/aldoc-expression-tracks-lamina-associated-chroma-4c06d0d43e_result.csv",
  "figure_path": "tmp/takei_auto_discovery_doc/run_pantheon_20_ideas_verified_agg/aldoc-expression-tracks-lamina-associated-chroma-4c06d0d43e_statistical_summary.png",
  "notes": [
    "Permutation p-value tests positive Spearman association by shuffling Aldoc labels across cells.",
    "Primary association pools all 9 cells; cell type is reported descriptively only because n=3 per type."
  ]
}

checks = {check: 'not_run' for check in IDEA.validation_checks}
notes = []
checks.setdefault('statistical_hypothesis_test', 'not_run')

def _check_keys(prefix):
    return [key for key in checks if key == prefix or key.startswith(prefix + ':')]

def _set_check(prefix, value):
    keys = _check_keys(prefix)
    if not keys:
        checks[prefix] = value
        return
    for key in keys:
        checks[key] = value

def _check_status(prefix):
    values = [checks[key] for key in _check_keys(prefix)]
    if not values:
        return None
    if 'fail' in values:
        return 'fail'
    if all(value == 'pass' for value in values):
        return 'pass'
    return values[0]

_set_check('required_fields_exist', 'pass' if review is not None and review.accepted else 'fail')
if _check_keys('cell_id_alignment'):
    aligned = True
    if cdata is not None and adata is not None and len(cdata.cells) == len(adata.obs_names):
        aligned = list(map(str, cdata.cells.index)) == list(map(str, adata.obs_names))
    _set_check('cell_id_alignment', 'pass' if aligned else 'fail')
if _check_keys('minimum_cell_count'):
    n_cells = analysis_summary.get('n_selected_cells')
    if n_cells is None and 'cell_type' in getattr(result_table, 'columns', []):
        n_cells = len(result_table)
    if n_cells is None:
        n_cells = len(cdata.cells) if cdata is not None and getattr(cdata, 'n_cells', 0) else 0
    _set_check('minimum_cell_count', 'pass' if n_cells >= 1 else 'fail')
if _check_keys('minimum_spot_or_trace_count'):
    n_rows = analysis_summary.get('n_rows')
    if n_rows is None:
        n_rows = len(result_table) if result_table is not None else 0
    _set_check('minimum_spot_or_trace_count', 'pass' if n_rows >= 1 else 'fail')
if _check_keys('finite_numeric_output'):
    value = analysis_summary.get('parameter_value')
    _set_check('finite_numeric_output', 'pass' if value is not None and np.isfinite(value) else 'fail')
if _check_keys('statistical_hypothesis_test'):
    p_value = analysis_summary.get('p_value')
    test_method = analysis_summary.get('test_method')
    null_hypothesis = analysis_summary.get('null_hypothesis')
    alternative_hypothesis = analysis_summary.get('alternative_hypothesis')
    observed_statistic = analysis_summary.get('observed_statistic')
    effect_size = analysis_summary.get('effect_size')
    hypothesis_test_status = analysis_summary.get('hypothesis_test_status', 'pass')
    try:
        p_float = float(p_value)
    except Exception:
        p_float = np.nan
    try:
        stat_float = float(observed_statistic)
    except Exception:
        stat_float = np.nan
    try:
        effect_float = float(effect_size)
    except Exception:
        effect_float = np.nan
    has_required_test = (
        test_method is not None
        and str(test_method).strip() != ''
        and null_hypothesis is not None
        and str(null_hypothesis).strip() != ''
        and alternative_hypothesis is not None
        and str(alternative_hypothesis).strip() != ''
        and np.isfinite(p_float)
        and 0.0 <= p_float <= 1.0
        and np.isfinite(stat_float)
        and np.isfinite(effect_float)
        and hypothesis_test_status != 'insufficient_data'
    )
    if result_table is not None and hasattr(result_table, 'columns'):
        has_required_test = has_required_test and 'p_value' in result_table.columns and 'test_method' in result_table.columns
    else:
        has_required_test = False
    _set_check('statistical_hypothesis_test', 'pass' if has_required_test else 'fail')
    if not has_required_test:
        notes.append('statistical_hypothesis_test failed: analysis_summary must include null_hypothesis, alternative_hypothesis, test_method, observed_statistic, effect_size, finite p_value in [0,1], and result_table columns p_value/test_method')
if _check_keys('negative_control_or_permutation'):
    test_method_text = str(analysis_summary.get('test_method', '')).lower()
    summary_keys_text = ' '.join(str(key).lower() for key in analysis_summary.keys())
    result_columns_text = ''
    if result_table is not None and hasattr(result_table, 'columns'):
        result_columns_text = ' '.join(str(col).lower() for col in result_table.columns)
    control_text = ' '.join([test_method_text, summary_keys_text, result_columns_text])
    has_control_or_permutation = any(
        token in control_text
        for token in ['permutation', 'randomization', 'shuffle', 'negative_control', 'null_distribution', 'control']
    )
    _set_check(
        'negative_control_or_permutation',
        'pass' if has_control_or_permutation else 'not_implemented',
    )
for check in list(checks):
    if checks[check] == 'not_run' and ('negative_control' in check or check.endswith('_control')):
        checks[check] = 'not_implemented'

required_for_pass = ['required_fields_exist', 'minimum_cell_count', 'finite_numeric_output', 'statistical_hypothesis_test']
status = 'pass'
for check in required_for_pass:
    if _check_status(check) == 'fail':
        status = 'fail'
        notes.append(f'{check} failed')
n_rows_for_status = analysis_summary.get('n_rows')
if n_rows_for_status is None:
    n_rows_for_status = len(result_table) if result_table is not None else 0
if n_rows_for_status == 0:
    status = 'fail'
    notes.append('analysis produced no result rows')

verification = {
    'idea_id': IDEA.idea_id,
    'status': status,
    'checks': checks,
    'parameter_value': analysis_summary.get('parameter_value'),
    'p_value': analysis_summary.get('p_value'),
    'test_method': analysis_summary.get('test_method'),
    'effect_size': analysis_summary.get('effect_size'),
    'result_path': analysis_summary.get('result_path'),
    'notes': notes + analysis_summary.get('notes', []),
}
print(json.dumps(verification, indent=2))

{
  "idea_id": "aldoc-expression-tracks-lamina-associated-chroma-4c06d0d43e",
  "status": "pass",
  "checks": {
    "required_fields_exist": "pass",
    "minimum_cell_count_n>=9_and_each_listed_cell_type_n>=3": "not_run",
    "minimum_spot_or_trace_count_per_cell_for_LaminB1_mean": "not_run",
    "finite_numeric_output": "pass",
    "statistical_hypothesis_test_spearman_with_p_value": "not_run",
    "runtime_under_budget": "not_run",
    "deterministic_rerun": "not_run",
    "negative_control_or_permutation_by_shuffling_Aldoc_expression_across_cells": "not_implemented",
    "statistical_hypothesis_test": "pass"
  },
  "parameter_value": 0.5355695416036628,
  "p_value": 0.07092907092907093,
  "test_method": "Spearman correlation with fixed-seed one-sided permutation test (1000 Aldoc-label shuffles)",
  "effect_size": 0.5355695416036628,
  "result_path": "tmp/takei_auto_discovery_doc/run_pantheon_20_ideas_verified_agg/aldoc-expression-tracks-lamina-associated-chroma-4c06d0d43e_result.csv",
  "notes": [
    "Permutation p-value tests positive Spearman association by shuffling Aldoc labels across cells.",
    "Primary association pools all 9 cells; cell type is reported descriptively only because n=3 per type."
  ]
}

Auto-discovery idea: Aldoc expression tracks lamina-associated chromatin signal¶

Rationale¶

Data used¶

Analysis sketch¶

Expected result¶

Validation checks¶

Graphical abstract¶

Required data checks¶

Exploration¶

Critique and compact analysis plan¶

Statistical figure¶

Runner verification summary¶

Final interpretation¶

Final interpretation¶