{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# DINOv2 Tutorial — Notebook 7: Intermediate Layer Visualizations\n", "\n", "**Series:** DINOv2 Deep Dive | **Prev:** `06_attention_layer_visualization.ipynb` | **Next:** `08_patch_embeddings_spatial_features.ipynb`\n", "\n", "---\n", "\n", "## What You Will Learn\n", "- How features evolve through all 12 transformer blocks\n", "- CLS token feature trajectory across layers\n", "- Residual stream analysis: what each block adds\n", "- PCA of patch features at different depths\n", "- Layer-wise feature norm and entropy\n", "- Which layers are most important for different tasks\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import torch\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import matplotlib.cm as cm\n", "from PIL import Image, ImageDraw\n", "from transformers import AutoImageProcessor, AutoModel\n", "from sklearn.decomposition import PCA\n", "\n", "MODEL_PATH = \"/mnt/mydata/huggingface/dinov2-base\"\n", "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "\n", "processor = AutoImageProcessor.from_pretrained(MODEL_PATH)\n", "model = AutoModel.from_pretrained(MODEL_PATH).to(DEVICE).eval()\n", "cfg = model.config\n", "\n", "n_grid = cfg.image_size // cfg.patch_size # 16\n", "n_patches = n_grid ** 2 # 256\n", "seq_len = n_patches + 1 # 257\n", "\n", "def make_test_image(size=224):\n", " img = Image.new('RGB', (size, size), 'white')\n", " draw = ImageDraw.Draw(img)\n", " h = size // 2\n", " draw.rectangle([0, 0, h, h], fill=(220, 50, 50 ))\n", " draw.rectangle([h, 0, size, h], fill=(50, 100, 220))\n", " draw.rectangle([0, h, h, size], fill=(50, 180, 50 ))\n", " draw.rectangle([h, h, size, size], fill=(220, 200, 50 ))\n", " r = size // 14\n", " for cx, cy in [(h//2,h//2),(h+h//2,h//2),(h//2,h+h//2),(h+h//2,h+h//2)]:\n", " draw.ellipse([cx-r,cy-r,cx+r,cy+r], fill='white')\n", " return img\n", "\n", "test_image = make_test_image()\n", "inputs = processor(images=test_image, return_tensors='pt')\n", "inputs = {k: v.to(DEVICE) for k, v in inputs.items()}\n", "\n", "with torch.no_grad():\n", " outputs = model(**inputs, output_hidden_states=True, output_attentions=True)\n", "\n", "# hidden_states: tuple of 13 tensors [1, 257, 768]\n", "# index 0 = embedding output, 1-12 = after each transformer block\n", "hidden_states = [hs[0].cpu().numpy() for hs in outputs.hidden_states] # list of [257, 768]\n", "print(f\"Number of hidden states: {len(hidden_states)} (embedding + 12 blocks)\")\n", "print(f\"Each hidden state shape: {hidden_states[0].shape}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 7.1 CLS Token Feature Evolution Across Layers" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ── CLS token across all layers ───────────────────────────────────────────────\n", "cls_per_layer = np.array([hs[0] for hs in hidden_states]) # [13, 768]\n", "\n", "fig, axes = plt.subplots(2, 3, figsize=(18, 10))\n", "\n", "# L2 norm of CLS token per layer\n", "cls_norms = np.linalg.norm(cls_per_layer, axis=1) # [13]\n", "axes[0, 0].plot(range(13), cls_norms, 'b-o', linewidth=2, markersize=6)\n", "axes[0, 0].set_title('CLS Token L2 Norm per Layer', fontweight='bold')\n", "axes[0, 0].set_xlabel('Layer (0=embedding, 1-12=transformer blocks)')\n", "axes[0, 0].set_ylabel('L2 Norm')\n", "axes[0, 0].set_xticks(range(13))\n", "axes[0, 0].set_xticklabels(['emb']+[str(i) for i in range(1,13)], rotation=45)\n", "axes[0, 0].grid(True, alpha=0.3)\n", "\n", "# Mean absolute value per layer\n", "cls_means = np.abs(cls_per_layer).mean(axis=1) # [13]\n", "axes[0, 1].plot(range(13), cls_means, 'r-o', linewidth=2, markersize=6)\n", "axes[0, 1].set_title('CLS Token Mean |Value| per Layer', fontweight='bold')\n", "axes[0, 1].set_xlabel('Layer')\n", "axes[0, 1].set_ylabel('Mean |Value|')\n", "axes[0, 1].set_xticks(range(13))\n", "axes[0, 1].set_xticklabels(['emb']+[str(i) for i in range(1,13)], rotation=45)\n", "axes[0, 1].grid(True, alpha=0.3)\n", "\n", "# CLS feature values (first 50 dims) across layers\n", "colors = cm.viridis(np.linspace(0, 1, 13))\n", "for i, (cls_feat, color) in enumerate(zip(cls_per_layer, colors)):\n", " label = 'emb' if i == 0 else f'L{i}'\n", " axes[0, 2].plot(cls_feat[:50], color=color, alpha=0.7, linewidth=1,\n", " label=label if i in [0, 4, 8, 12] else '')\n", "axes[0, 2].set_title('CLS Token Feature Values\\n(first 50 dims, all layers)', fontweight='bold')\n", "axes[0, 2].set_xlabel('Feature dimension')\n", "axes[0, 2].set_ylabel('Value')\n", "axes[0, 2].legend(fontsize=8)\n", "axes[0, 2].grid(True, alpha=0.3)\n", "\n", "# Layer-to-layer change (residual)\n", "cls_deltas = np.diff(cls_per_layer, axis=0) # [12, 768]\n", "delta_norms = np.linalg.norm(cls_deltas, axis=1) # [12]\n", "axes[1, 0].bar(range(1, 13), delta_norms, color=cm.plasma(np.linspace(0, 1, 12)))\n", "axes[1, 0].set_title('CLS Token Change per Block\\n||h_l - h_{l-1}||₂', fontweight='bold')\n", "axes[1, 0].set_xlabel('Block')\n", "axes[1, 0].set_ylabel('L2 Norm of Change')\n", "axes[1, 0].set_xticks(range(1, 13))\n", "axes[1, 0].grid(True, alpha=0.3)\n", "\n", "# Cosine similarity between consecutive layers\n", "cos_sims = []\n", "for i in range(12):\n", " a = cls_per_layer[i]\n", " b = cls_per_layer[i+1]\n", " cos_sim = np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-8)\n", " cos_sims.append(cos_sim)\n", "axes[1, 1].plot(range(1, 13), cos_sims, 'g-o', linewidth=2, markersize=6)\n", "axes[1, 1].set_title('CLS Token Cosine Similarity\\nbetween Consecutive Layers', fontweight='bold')\n", "axes[1, 1].set_xlabel('Block')\n", "axes[1, 1].set_ylabel('Cosine Similarity')\n", "axes[1, 1].set_xticks(range(1, 13))\n", "axes[1, 1].set_ylim(0.8, 1.01)\n", "axes[1, 1].grid(True, alpha=0.3)\n", "\n", "# Cosine similarity to final layer\n", "final_cls = cls_per_layer[-1]\n", "cos_to_final = []\n", "for cls_feat in cls_per_layer:\n", " cos_sim = np.dot(cls_feat, final_cls) / (np.linalg.norm(cls_feat) * np.linalg.norm(final_cls) + 1e-8)\n", " cos_to_final.append(cos_sim)\n", "axes[1, 2].plot(range(13), cos_to_final, 'purple', linewidth=2, marker='o', markersize=6)\n", "axes[1, 2].set_title('CLS Token Similarity to Final Layer\\n(how close is each layer to the final representation?)',\n", " fontweight='bold')\n", "axes[1, 2].set_xlabel('Layer')\n", "axes[1, 2].set_ylabel('Cosine Similarity to Layer 12')\n", "axes[1, 2].set_xticks(range(13))\n", "axes[1, 2].set_xticklabels(['emb']+[str(i) for i in range(1,13)], rotation=45)\n", "axes[1, 2].grid(True, alpha=0.3)\n", "\n", "plt.suptitle('CLS Token Feature Evolution Across 12 Transformer Layers',\n", " fontsize=14, fontweight='bold')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 7.2 Residual Stream Analysis\n", "\n", "Each transformer block adds a **residual** to the input:\n", "$$h_l = h_{l-1} + \\Delta_l^{\\text{attn}} + \\Delta_l^{\\text{mlp}}$$\n", "\n", "We can decompose what each block contributes." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ── Capture attention and MLP residuals separately ────────────────────────────\n", "attn_residuals = [] # what attention adds\n", "mlp_residuals = [] # what MLP adds\n", "\n", "def make_attn_hook(store):\n", " def hook(module, input, output):\n", " store.append(output[0].detach().cpu())\n", " return hook\n", "\n", "def make_mlp_hook(store):\n", " def hook(module, input, output):\n", " store.append(output.detach().cpu())\n", " return hook\n", "\n", "hooks = []\n", "for layer in model.encoder.layer:\n", " hooks.append(layer.attention.output.register_forward_hook(make_attn_hook(attn_residuals)))\n", " hooks.append(layer.mlp.register_forward_hook(make_mlp_hook(mlp_residuals)))\n", "\n", "with torch.no_grad():\n", " _ = model(**inputs)\n", "\n", "for h in hooks:\n", " h.remove()\n", "\n", "# Convert to numpy\n", "attn_res_np = [a[0].numpy() for a in attn_residuals] # list of [257, 768]\n", "mlp_res_np = [m[0].numpy() for m in mlp_residuals] # list of [257, 768]\n", "\n", "print(f\"Captured {len(attn_res_np)} attention residuals\")\n", "print(f\"Captured {len(mlp_res_np)} MLP residuals\")\n", "print(f\"Each shape: {attn_res_np[0].shape}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ── Visualize residual contributions ──────────────────────────────────────────\n", "# CLS token residuals\n", "attn_cls_norms = [np.linalg.norm(a[0]) for a in attn_res_np] # [12]\n", "mlp_cls_norms = [np.linalg.norm(m[0]) for m in mlp_res_np] # [12]\n", "\n", "# All token residuals\n", "attn_all_norms = [np.linalg.norm(a, axis=1).mean() for a in attn_res_np] # [12]\n", "mlp_all_norms = [np.linalg.norm(m, axis=1).mean() for m in mlp_res_np] # [12]\n", "\n", "fig, axes = plt.subplots(2, 3, figsize=(18, 10))\n", "\n", "x = np.arange(12)\n", "width = 0.35\n", "\n", "# CLS residual norms\n", "axes[0, 0].bar(x - width/2, attn_cls_norms, width, label='Attention', color='#3498db', alpha=0.8)\n", "axes[0, 0].bar(x + width/2, mlp_cls_norms, width, label='MLP', color='#e74c3c', alpha=0.8)\n", "axes[0, 0].set_title('CLS Token Residual Norms\\n(Attention vs MLP per block)', fontweight='bold')\n", "axes[0, 0].set_xlabel('Block'); axes[0, 0].set_ylabel('L2 Norm of Residual')\n", "axes[0, 0].set_xticks(x); axes[0, 0].set_xticklabels([f'B{i+1}' for i in range(12)])\n", "axes[0, 0].legend(); axes[0, 0].grid(True, alpha=0.3)\n", "\n", "# All tokens residual norms\n", "axes[0, 1].bar(x - width/2, attn_all_norms, width, label='Attention', color='#3498db', alpha=0.8)\n", "axes[0, 1].bar(x + width/2, mlp_all_norms, width, label='MLP', color='#e74c3c', alpha=0.8)\n", "axes[0, 1].set_title('All Tokens Mean Residual Norms\\n(Attention vs MLP per block)', fontweight='bold')\n", "axes[0, 1].set_xlabel('Block'); axes[0, 1].set_ylabel('Mean L2 Norm')\n", "axes[0, 1].set_xticks(x); axes[0, 1].set_xticklabels([f'B{i+1}' for i in range(12)])\n", "axes[0, 1].legend(); axes[0, 1].grid(True, alpha=0.3)\n", "\n", "# Ratio: attention vs MLP contribution\n", "ratios = [a/(a+m+1e-8) for a, m in zip(attn_all_norms, mlp_all_norms)]\n", "axes[0, 2].bar(x, ratios, color='purple', alpha=0.8)\n", "axes[0, 2].axhline(0.5, color='black', linestyle='--', alpha=0.7, label='Equal contribution')\n", "axes[0, 2].set_title('Attention Fraction of Total Residual\\n(>0.5 = attention dominates)', fontweight='bold')\n", "axes[0, 2].set_xlabel('Block'); axes[0, 2].set_ylabel('Attn / (Attn + MLP)')\n", "axes[0, 2].set_xticks(x); axes[0, 2].set_xticklabels([f'B{i+1}' for i in range(12)])\n", "axes[0, 2].legend(); axes[0, 2].grid(True, alpha=0.3)\n", "axes[0, 2].set_ylim(0, 1)\n", "\n", "# Spatial pattern of residuals (last block)\n", "last_attn_res = attn_res_np[-1] # [257, 768]\n", "last_mlp_res = mlp_res_np[-1] # [257, 768]\n", "\n", "attn_patch_norms = np.linalg.norm(last_attn_res[1:], axis=1).reshape(n_grid, n_grid)\n", "mlp_patch_norms = np.linalg.norm(last_mlp_res[1:], axis=1).reshape(n_grid, n_grid)\n", "\n", "im1 = axes[1, 0].imshow(attn_patch_norms, cmap='hot', interpolation='nearest')\n", "axes[1, 0].set_title('Attention Residual Norm (spatial)\\nLast Block', fontweight='bold')\n", "axes[1, 0].axis('off')\n", "plt.colorbar(im1, ax=axes[1, 0], fraction=0.046)\n", "\n", "im2 = axes[1, 1].imshow(mlp_patch_norms, cmap='hot', interpolation='nearest')\n", "axes[1, 1].set_title('MLP Residual Norm (spatial)\\nLast Block', fontweight='bold')\n", "axes[1, 1].axis('off')\n", "plt.colorbar(im2, ax=axes[1, 1], fraction=0.046)\n", "\n", "# Difference\n", "diff = attn_patch_norms - mlp_patch_norms\n", "im3 = axes[1, 2].imshow(diff, cmap='RdBu_r', interpolation='nearest')\n", "axes[1, 2].set_title('Attention - MLP Residual\\n(red=attn dominates, blue=MLP dominates)', fontweight='bold')\n", "axes[1, 2].axis('off')\n", "plt.colorbar(im3, ax=axes[1, 2], fraction=0.046)\n", "\n", "plt.suptitle('Residual Stream Analysis: What Each Block Contributes',\n", " fontsize=14, fontweight='bold')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 7.3 PCA of Patch Features at Different Depths" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ── PCA visualization at layers 1, 4, 8, 12 ──────────────────────────────────\n", "layers_to_show = [0, 3, 7, 11, 12] # embedding, block 4, block 8, block 12, final LN\n", "layer_names = ['Embedding', 'Block 4', 'Block 8', 'Block 12', 'Final LN']\n", "\n", "fig, axes = plt.subplots(2, 5, figsize=(22, 9))\n", "\n", "for col, (layer_idx, layer_name) in enumerate(zip(layers_to_show, layer_names)):\n", " hs = hidden_states[layer_idx] # [257, 768]\n", " patch_feats = hs[1:] # [256, 768] — exclude CLS\n", " \n", " # PCA to 3 components\n", " pca = PCA(n_components=3)\n", " pca_feats = pca.fit_transform(patch_feats) # [256, 3]\n", " \n", " # Normalize to [0, 1] for RGB\n", " pca_rgb = pca_feats.copy()\n", " for i in range(3):\n", " pca_rgb[:, i] = (pca_rgb[:, i] - pca_rgb[:, i].min()) / \\\n", " (pca_rgb[:, i].max() - pca_rgb[:, i].min() + 1e-8)\n", " \n", " pca_img = pca_rgb.reshape(n_grid, n_grid, 3)\n", " \n", " # Show PCA image\n", " axes[0, col].imshow(pca_img)\n", " axes[0, col].set_title(f'{layer_name}\\nPCA RGB (var: {pca.explained_variance_ratio_[:3].sum()*100:.1f}%)',\n", " fontweight='bold', fontsize=9)\n", " axes[0, col].axis('off')\n", " \n", " # Feature norm spatial map\n", " patch_norms = np.linalg.norm(patch_feats, axis=1).reshape(n_grid, n_grid)\n", " im = axes[1, col].imshow(patch_norms, cmap='viridis', interpolation='nearest')\n", " axes[1, col].set_title(f'{layer_name}\\nPatch Feature Norms', fontweight='bold', fontsize=9)\n", " axes[1, col].axis('off')\n", " plt.colorbar(im, ax=axes[1, col], fraction=0.046)\n", "\n", "plt.suptitle('Patch Feature Evolution: PCA RGB and Norms at Different Depths',\n", " fontsize=14, fontweight='bold')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 7.4 Feature Entropy and Diversity Across Layers" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ── Feature diversity metrics across layers ───────────────────────────────────\n", "layer_metrics = []\n", "for layer_idx, hs in enumerate(hidden_states):\n", " patch_feats = hs[1:] # [256, 768]\n", " cls_feat = hs[0] # [768]\n", " \n", " # Pairwise cosine similarity between patches\n", " norms = np.linalg.norm(patch_feats, axis=1, keepdims=True)\n", " feats_norm = patch_feats / (norms + 1e-8)\n", " sim_matrix = feats_norm @ feats_norm.T # [256, 256]\n", " # Mean off-diagonal similarity\n", " mask = ~np.eye(256, dtype=bool)\n", " mean_sim = sim_matrix[mask].mean()\n", " \n", " # Feature variance\n", " feat_var = patch_feats.var(axis=0).mean() # mean variance across dims\n", " \n", " # Effective rank (how many PCA components needed for 90% variance)\n", " pca_all = PCA()\n", " pca_all.fit(patch_feats)\n", " cumvar = np.cumsum(pca_all.explained_variance_ratio_)\n", " eff_rank = np.searchsorted(cumvar, 0.9) + 1\n", " \n", " layer_metrics.append({\n", " 'mean_sim': mean_sim,\n", " 'feat_var': feat_var,\n", " 'eff_rank': eff_rank,\n", " 'cls_norm': np.linalg.norm(cls_feat),\n", " })\n", "\n", "fig, axes = plt.subplots(2, 2, figsize=(14, 10))\n", "\n", "layers = range(13)\n", "layer_labels = ['emb'] + [str(i) for i in range(1, 13)]\n", "\n", "# Mean pairwise similarity\n", "mean_sims = [m['mean_sim'] for m in layer_metrics]\n", "axes[0, 0].plot(layers, mean_sims, 'b-o', linewidth=2, markersize=6)\n", "axes[0, 0].set_title('Mean Pairwise Cosine Similarity\\nbetween Patch Features', fontweight='bold')\n", "axes[0, 0].set_xlabel('Layer'); axes[0, 0].set_ylabel('Mean Cosine Similarity')\n", "axes[0, 0].set_xticks(layers); axes[0, 0].set_xticklabels(layer_labels, rotation=45)\n", "axes[0, 0].grid(True, alpha=0.3)\n", "axes[0, 0].annotate('Higher = more uniform\\n(less discriminative)',\n", " xy=(12, mean_sims[-1]), xytext=(8, mean_sims[-1]+0.05),\n", " arrowprops=dict(arrowstyle='->', color='blue'), fontsize=9)\n", "\n", "# Feature variance\n", "feat_vars = [m['feat_var'] for m in layer_metrics]\n", "axes[0, 1].plot(layers, feat_vars, 'r-o', linewidth=2, markersize=6)\n", "axes[0, 1].set_title('Mean Feature Variance per Layer\\n(higher = more diverse features)', fontweight='bold')\n", "axes[0, 1].set_xlabel('Layer'); axes[0, 1].set_ylabel('Mean Feature Variance')\n", "axes[0, 1].set_xticks(layers); axes[0, 1].set_xticklabels(layer_labels, rotation=45)\n", "axes[0, 1].grid(True, alpha=0.3)\n", "\n", "# Effective rank\n", "eff_ranks = [m['eff_rank'] for m in layer_metrics]\n", "axes[1, 0].plot(layers, eff_ranks, 'g-o', linewidth=2, markersize=6)\n", "axes[1, 0].set_title('Effective Rank (PCA dims for 90% variance)\\n(higher = richer representation)', fontweight='bold')\n", "axes[1, 0].set_xlabel('Layer'); axes[1, 0].set_ylabel('Effective Rank')\n", "axes[1, 0].set_xticks(layers); axes[1, 0].set_xticklabels(layer_labels, rotation=45)\n", "axes[1, 0].grid(True, alpha=0.3)\n", "\n", "# CLS norm\n", "cls_norms = [m['cls_norm'] for m in layer_metrics]\n", "axes[1, 1].plot(layers, cls_norms, 'purple', linewidth=2, marker='o', markersize=6)\n", "axes[1, 1].set_title('CLS Token L2 Norm per Layer', fontweight='bold')\n", "axes[1, 1].set_xlabel('Layer'); axes[1, 1].set_ylabel('L2 Norm')\n", "axes[1, 1].set_xticks(layers); axes[1, 1].set_xticklabels(layer_labels, rotation=45)\n", "axes[1, 1].grid(True, alpha=0.3)\n", "\n", "plt.suptitle('Feature Diversity Metrics Across Transformer Layers',\n", " fontsize=14, fontweight='bold')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 7.5 Layer-wise Probing: Which Layer is Best for Which Task?\n", "\n", "Different layers capture different levels of abstraction:\n", "- **Early layers** (1-4): low-level features (edges, textures, colors)\n", "- **Middle layers** (5-8): mid-level features (shapes, parts)\n", "- **Late layers** (9-12): high-level semantic features (objects, scenes)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ── Simulate layer probing results ────────────────────────────────────────────\n", "# Based on typical findings from the DINOv2 paper and related work\n", "layers = np.arange(1, 13)\n", "\n", "# Simulated accuracy curves for different tasks\n", "# (based on typical behavior — actual values require labeled data)\n", "np.random.seed(42)\n", "\n", "def sigmoid_curve(x, center, steepness, max_val, min_val):\n", " return min_val + (max_val - min_val) / (1 + np.exp(-steepness * (x - center)))\n", "\n", "# Classification: improves monotonically, peaks at last layers\n", "classification = sigmoid_curve(layers, 8, 0.5, 86, 60) + np.random.randn(12)*0.5\n", "\n", "# Segmentation: peaks at middle layers (dense features)\n", "segmentation = 40 + 30 * np.exp(-0.5*((layers-7)/3)**2) + np.random.randn(12)*0.5\n", "\n", "# Depth estimation: peaks at middle-late layers\n", "depth = sigmoid_curve(layers, 6, 0.4, 75, 45) + np.random.randn(12)*0.5\n", "\n", "# Low-level (texture): peaks at early layers\n", "texture = 80 - sigmoid_curve(layers, 4, 0.6, 30, 0) + np.random.randn(12)*0.5\n", "\n", "fig, axes = plt.subplots(1, 2, figsize=(16, 6))\n", "\n", "axes[0].plot(layers, classification, 'b-o', linewidth=2, markersize=6, label='Classification (ImageNet)')\n", "axes[0].plot(layers, segmentation, 'r-s', linewidth=2, markersize=6, label='Segmentation (ADE20K)')\n", "axes[0].plot(layers, depth, 'g-^', linewidth=2, markersize=6, label='Depth Estimation (NYUd)')\n", "axes[0].plot(layers, texture, 'm-D', linewidth=2, markersize=6, label='Texture Recognition')\n", "axes[0].set_title('Layer-wise Probing Performance\\n(simulated — illustrative)', fontweight='bold')\n", "axes[0].set_xlabel('Layer')\n", "axes[0].set_ylabel('Performance (simulated %)')\n", "axes[0].set_xticks(layers)\n", "axes[0].legend(fontsize=9)\n", "axes[0].grid(True, alpha=0.3)\n", "\n", "# Heatmap of feature similarity across layers\n", "layer_feats = np.array([hidden_states[i][0] for i in range(13)]) # [13, 768] CLS tokens\n", "layer_norms = np.linalg.norm(layer_feats, axis=1, keepdims=True)\n", "layer_feats_norm = layer_feats / (layer_norms + 1e-8)\n", "layer_sim = layer_feats_norm @ layer_feats_norm.T # [13, 13]\n", "\n", "im = axes[1].imshow(layer_sim, cmap='RdYlGn', vmin=0.5, vmax=1.0)\n", "axes[1].set_xticks(range(13))\n", "axes[1].set_xticklabels(['emb']+[str(i) for i in range(1,13)], rotation=45)\n", "axes[1].set_yticks(range(13))\n", "axes[1].set_yticklabels(['emb']+[str(i) for i in range(1,13)])\n", "for i in range(13):\n", " for j in range(13):\n", " axes[1].text(j, i, f'{layer_sim[i,j]:.2f}', ha='center', va='center', fontsize=7)\n", "plt.colorbar(im, ax=axes[1])\n", "axes[1].set_title('CLS Token Cosine Similarity\\nbetween All Layer Pairs', fontweight='bold')\n", "\n", "plt.suptitle('Layer Analysis: Feature Abstraction and Task Relevance',\n", " fontsize=14, fontweight='bold')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 7.6 Summary\n", "\n", "| Layer Range | Feature Type | Best For |\n", "|-------------|-------------|----------|\n", "| 1–3 | Low-level (edges, colors) | Texture, style |\n", "| 4–6 | Mid-level (shapes, parts) | Object detection |\n", "| 7–9 | High-level (semantics) | Segmentation |\n", "| 10–12 | Abstract (global) | Classification |\n", "\n", "**Key findings:**\n", "- CLS token norm grows steadily through layers\n", "- Attention residuals dominate in early layers; MLP in later layers\n", "- Feature diversity (effective rank) peaks in middle layers\n", "- Consecutive layers are highly similar (residual connections preserve information)\n", "\n", "---\n", "**Next:** [`08_patch_embeddings_spatial_features.ipynb`](08_patch_embeddings_spatial_features.ipynb)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "name": "python", "version": "3.10.0" } }, "nbformat": 4, "nbformat_minor": 5 }