{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# DINOv2 Tutorial — Notebook 8: Patch Embeddings & Spatial Feature Maps\n", "\n", "**Series:** DINOv2 Deep Dive | **Prev:** `07_intermediate_layer_visualization.ipynb` | **Next:** `09_cls_token_analysis.ipynb`\n", "\n", "---\n", "\n", "## What You Will Learn\n", "- How to extract and visualize patch-level features\n", "- PCA of patch features → semantic segmentation-like maps\n", "- Feature heatmaps: mean, std, L2-norm per patch\n", "- K-means clustering of patch features\n", "- Upsampling patch features to full image resolution\n", "- Comparing patch features across different images\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import torch\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import matplotlib.cm as cm\n", "from PIL import Image, ImageDraw\n", "from transformers import AutoImageProcessor, AutoModel\n", "from sklearn.decomposition import PCA\n", "from sklearn.cluster import KMeans\n", "\n", "MODEL_PATH = \"/mnt/mydata/huggingface/dinov2-base\"\n", "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "\n", "processor = AutoImageProcessor.from_pretrained(MODEL_PATH)\n", "model = AutoModel.from_pretrained(MODEL_PATH).to(DEVICE).eval()\n", "cfg = model.config\n", "\n", "n_grid = cfg.image_size // cfg.patch_size # 16\n", "n_patches = n_grid ** 2 # 256\n", "\n", "IMAGENET_MEAN = np.array([0.485, 0.456, 0.406])\n", "IMAGENET_STD = np.array([0.229, 0.224, 0.225])\n", "\n", "def extract_patch_features(image):\n", " \"\"\"Extract patch features [n_patches, hidden_dim] from a PIL image.\"\"\"\n", " inp = processor(images=image, return_tensors='pt')\n", " inp = {k: v.to(DEVICE) for k, v in inp.items()}\n", " with torch.no_grad():\n", " out = model(**inp)\n", " return out.last_hidden_state[0, 1:].cpu().numpy() # [256, 768]\n", "\n", "def extract_cls_feature(image):\n", " \"\"\"Extract CLS token feature [hidden_dim] from a PIL image.\"\"\"\n", " inp = processor(images=image, return_tensors='pt')\n", " inp = {k: v.to(DEVICE) for k, v in inp.items()}\n", " with torch.no_grad():\n", " out = model(**inp)\n", " return out.last_hidden_state[0, 0].cpu().numpy() # [768]\n", "\n", "def denormalize(tensor):\n", " \"\"\"Denormalize [3,H,W] tensor to [H,W,3] numpy array in [0,1].\"\"\"\n", " t = tensor.cpu().numpy()\n", " t = t.transpose(1, 2, 0)\n", " t = t * IMAGENET_STD + IMAGENET_MEAN\n", " return t.clip(0, 1)\n", "\n", "# Create test images\n", "def make_test_image(size=224):\n", " img = Image.new('RGB', (size, size), 'white')\n", " draw = ImageDraw.Draw(img)\n", " h = size // 2\n", " draw.rectangle([0, 0, h, h], fill=(220, 50, 50 ))\n", " draw.rectangle([h, 0, size, h], fill=(50, 100, 220))\n", " draw.rectangle([0, h, h, size], fill=(50, 180, 50 ))\n", " draw.rectangle([h, h, size, size], fill=(220, 200, 50 ))\n", " r = size // 14\n", " for cx, cy in [(h//2,h//2),(h+h//2,h//2),(h//2,h+h//2),(h+h//2,h+h//2)]:\n", " draw.ellipse([cx-r,cy-r,cx+r,cy+r], fill='white')\n", " return img\n", "\n", "test_image = make_test_image()\n", "patch_feats = extract_patch_features(test_image) # [256, 768]\n", "print(f\"Patch features shape: {patch_feats.shape}\")\n", "print(f\" = [{n_patches} patches, {cfg.hidden_size} dims]\")\n", "print(f\" = [{n_grid}×{n_grid} spatial grid, {cfg.hidden_size} dims]\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 8.1 Basic Patch Feature Statistics" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ── Patch feature statistics ───────────────────────────────────────────────────\n", "patch_means = patch_feats.mean(axis=1) # [256] — mean value per patch\n", "patch_stds = patch_feats.std(axis=1) # [256] — std per patch\n", "patch_norms = np.linalg.norm(patch_feats, axis=1) # [256] — L2 norm per patch\n", "patch_maxes = patch_feats.max(axis=1) # [256] — max activation per patch\n", "\n", "# Reshape to spatial grids\n", "mean_grid = patch_means.reshape(n_grid, n_grid)\n", "std_grid = patch_stds.reshape(n_grid, n_grid)\n", "norm_grid = patch_norms.reshape(n_grid, n_grid)\n", "max_grid = patch_maxes.reshape(n_grid, n_grid)\n", "\n", "fig, axes = plt.subplots(2, 3, figsize=(18, 12))\n", "\n", "# Original image\n", "axes[0, 0].imshow(test_image)\n", "# Overlay patch grid\n", "for i in range(0, cfg.image_size + 1, cfg.patch_size):\n", " axes[0, 0].axhline(i, color='white', linewidth=0.5, alpha=0.6)\n", " axes[0, 0].axvline(i, color='white', linewidth=0.5, alpha=0.6)\n", "axes[0, 0].set_title(f'Original Image\\n{n_grid}×{n_grid} patch grid overlay', fontweight='bold')\n", "axes[0, 0].axis('off')\n", "\n", "# Mean feature value\n", "im1 = axes[0, 1].imshow(mean_grid, cmap='RdBu_r', interpolation='nearest')\n", "axes[0, 1].set_title('Mean Feature Value per Patch\\n(positive=active, negative=suppressed)',\n", " fontweight='bold')\n", "axes[0, 1].axis('off')\n", "plt.colorbar(im1, ax=axes[0, 1], fraction=0.046)\n", "\n", "# Std dev\n", "im2 = axes[0, 2].imshow(std_grid, cmap='plasma', interpolation='nearest')\n", "axes[0, 2].set_title('Feature Std Dev per Patch\\n(higher = more diverse activations)',\n", " fontweight='bold')\n", "axes[0, 2].axis('off')\n", "plt.colorbar(im2, ax=axes[0, 2], fraction=0.046)\n", "\n", "# L2 norm\n", "im3 = axes[1, 0].imshow(norm_grid, cmap='hot', interpolation='nearest')\n", "axes[1, 0].set_title('L2 Norm per Patch\\n(higher = stronger activation)',\n", " fontweight='bold')\n", "axes[1, 0].axis('off')\n", "plt.colorbar(im3, ax=axes[1, 0], fraction=0.046)\n", "\n", "# Max activation\n", "im4 = axes[1, 1].imshow(max_grid, cmap='viridis', interpolation='nearest')\n", "axes[1, 1].set_title('Max Feature Value per Patch\\n(peak activation)',\n", " fontweight='bold')\n", "axes[1, 1].axis('off')\n", "plt.colorbar(im4, ax=axes[1, 1], fraction=0.046)\n", "\n", "# Overlay norm on image\n", "norm_normalized = (norm_grid - norm_grid.min()) / (norm_grid.max() - norm_grid.min())\n", "norm_upsampled = np.array(Image.fromarray((norm_normalized * 255).astype(np.uint8)).resize(\n", " (cfg.image_size, cfg.image_size), Image.BILINEAR)) / 255.0\n", "heatmap = cm.hot(norm_upsampled)[:, :, :3]\n", "img_np = np.array(test_image) / 255.0\n", "overlay = 0.5 * img_np + 0.5 * heatmap\n", "axes[1, 2].imshow(overlay.clip(0, 1))\n", "axes[1, 2].set_title('L2 Norm Heatmap Overlay\\non Original Image', fontweight='bold')\n", "axes[1, 2].axis('off')\n", "\n", "plt.suptitle('Patch Feature Statistics — Spatial Maps', fontsize=14, fontweight='bold')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 8.2 PCA of Patch Features → Semantic Maps\n", "\n", "One of DINOv2's most impressive properties: **PCA of patch features reveals semantic structure** without any supervision.\n", "\n", "The first 3 PCA components can be mapped to RGB channels, producing a pseudo-segmentation map." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ── PCA visualization ─────────────────────────────────────────────────────────\n", "pca = PCA(n_components=10)\n", "pca_feats = pca.fit_transform(patch_feats) # [256, 10]\n", "\n", "print(f\"PCA explained variance:\")\n", "for i, var in enumerate(pca.explained_variance_ratio_[:10]):\n", " bar = '█' * int(var * 100)\n", " print(f\" PC{i+1:2d}: {var*100:5.1f}% {bar}\")\n", "print(f\" Total (10 PCs): {pca.explained_variance_ratio_[:10].sum()*100:.1f}%\")\n", "\n", "fig, axes = plt.subplots(2, 5, figsize=(22, 9))\n", "\n", "# First 5 PCA components as spatial maps\n", "for i in range(5):\n", " pc_map = pca_feats[:, i].reshape(n_grid, n_grid)\n", " im = axes[0, i].imshow(pc_map, cmap='RdBu_r', interpolation='bilinear')\n", " axes[0, i].set_title(f'PC{i+1}\\n({pca.explained_variance_ratio_[i]*100:.1f}% var)',\n", " fontweight='bold', fontsize=9)\n", " axes[0, i].axis('off')\n", " plt.colorbar(im, ax=axes[0, i], fraction=0.046)\n", "\n", "# RGB combinations\n", "combos = [\n", " ([0, 1, 2], 'PC1+PC2+PC3 (RGB)'),\n", " ([1, 2, 3], 'PC2+PC3+PC4 (RGB)'),\n", " ([0, 2, 4], 'PC1+PC3+PC5 (RGB)'),\n", "]\n", "\n", "for col, (pc_indices, title) in enumerate(combos):\n", " rgb = np.zeros((n_grid, n_grid, 3))\n", " for ch, pc_idx in enumerate(pc_indices):\n", " pc_map = pca_feats[:, pc_idx].reshape(n_grid, n_grid)\n", " rgb[:, :, ch] = (pc_map - pc_map.min()) / (pc_map.max() - pc_map.min() + 1e-8)\n", " axes[1, col].imshow(rgb)\n", " axes[1, col].set_title(title, fontweight='bold', fontsize=9)\n", " axes[1, col].axis('off')\n", "\n", "# Original image\n", "axes[1, 3].imshow(test_image)\n", "axes[1, 3].set_title('Original Image', fontweight='bold')\n", "axes[1, 3].axis('off')\n", "\n", "# Scree plot\n", "axes[1, 4].bar(range(1, 11), pca.explained_variance_ratio_[:10] * 100,\n", " color=cm.viridis(np.linspace(0, 1, 10)))\n", "axes[1, 4].plot(range(1, 11), np.cumsum(pca.explained_variance_ratio_[:10]) * 100,\n", " 'r-o', linewidth=2, markersize=5, label='Cumulative')\n", "axes[1, 4].set_title('PCA Scree Plot\\n(% variance explained)', fontweight='bold', fontsize=9)\n", "axes[1, 4].set_xlabel('PC'); axes[1, 4].set_ylabel('Variance (%)')\n", "axes[1, 4].legend(fontsize=8); axes[1, 4].grid(True, alpha=0.3)\n", "\n", "plt.suptitle('PCA of Patch Features — Semantic Structure Emerges Without Supervision',\n", " fontsize=14, fontweight='bold')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 8.3 K-Means Clustering of Patch Features" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ── K-means clustering ────────────────────────────────────────────────────────\n", "fig, axes = plt.subplots(2, 4, figsize=(20, 10))\n", "\n", "k_values = [2, 3, 4, 5]\n", "cluster_colors = [\n", " ['#e74c3c', '#3498db'],\n", " ['#e74c3c', '#3498db', '#2ecc71'],\n", " ['#e74c3c', '#3498db', '#2ecc71', '#f39c12'],\n", " ['#e74c3c', '#3498db', '#2ecc71', '#f39c12', '#9b59b6'],\n", "]\n", "\n", "# Original image\n", "axes[0, 0].imshow(test_image)\n", "axes[0, 0].set_title('Original Image', fontweight='bold')\n", "axes[0, 0].axis('off')\n", "\n", "for col, (k, colors) in enumerate(zip(k_values, cluster_colors)):\n", " kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)\n", " labels = kmeans.fit_predict(patch_feats) # [256]\n", " \n", " # Create color map\n", " label_grid = labels.reshape(n_grid, n_grid)\n", " \n", " # RGB image from cluster labels\n", " cluster_img = np.zeros((n_grid, n_grid, 3))\n", " for cluster_id, color_hex in enumerate(colors):\n", " r = int(color_hex[1:3], 16) / 255\n", " g = int(color_hex[3:5], 16) / 255\n", " b = int(color_hex[5:7], 16) / 255\n", " mask = label_grid == cluster_id\n", " cluster_img[mask] = [r, g, b]\n", " \n", " axes[0, col+1 if col < 3 else col].imshow(cluster_img, interpolation='nearest')\n", " axes[0, col+1 if col < 3 else col].set_title(f'K-Means (k={k})\\nPatch Clusters',\n", " fontweight='bold', fontsize=9)\n", " axes[0, col+1 if col < 3 else col].axis('off')\n", "\n", "# Upsampled cluster maps\n", "for col, (k, colors) in enumerate(zip(k_values, cluster_colors)):\n", " kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)\n", " labels = kmeans.fit_predict(patch_feats)\n", " label_grid = labels.reshape(n_grid, n_grid)\n", " \n", " # Upsample to image size\n", " label_upsampled = np.array(Image.fromarray(label_grid.astype(np.uint8)).resize(\n", " (cfg.image_size, cfg.image_size), Image.NEAREST))\n", " \n", " # Create overlay\n", " overlay = np.array(test_image).astype(float) / 255.0\n", " color_overlay = np.zeros_like(overlay)\n", " for cluster_id, color_hex in enumerate(colors):\n", " r = int(color_hex[1:3], 16) / 255\n", " g = int(color_hex[3:5], 16) / 255\n", " b = int(color_hex[5:7], 16) / 255\n", " mask = label_upsampled == cluster_id\n", " color_overlay[mask] = [r, g, b]\n", " \n", " blended = 0.5 * overlay + 0.5 * color_overlay\n", " axes[1, col].imshow(blended.clip(0, 1))\n", " axes[1, col].set_title(f'K-Means (k={k}) Overlay\\n(upsampled to 224×224)',\n", " fontweight='bold', fontsize=9)\n", " axes[1, col].axis('off')\n", "\n", "plt.suptitle('K-Means Clustering of Patch Features\\n'\n", " 'Unsupervised Semantic Segmentation',\n", " fontsize=14, fontweight='bold')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 8.4 Patch Feature Similarity Maps" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ── Patch-to-patch similarity ──────────────────────────────────────────────────\n", "# Normalize patch features\n", "patch_norms_vec = np.linalg.norm(patch_feats, axis=1, keepdims=True)\n", "patch_feats_norm = patch_feats / (patch_norms_vec + 1e-8)\n", "\n", "# Full similarity matrix\n", "sim_matrix = patch_feats_norm @ patch_feats_norm.T # [256, 256]\n", "\n", "# Pick query patches\n", "query_patches = [\n", " (n_grid//4, n_grid//4, 'Top-Left (red)'),\n", " (n_grid//4, 3*n_grid//4, 'Top-Right (blue)'),\n", " (3*n_grid//4, n_grid//4, 'Bottom-Left (green)'),\n", " (n_grid//2, n_grid//2, 'Center'),\n", "]\n", "\n", "fig, axes = plt.subplots(2, 5, figsize=(22, 9))\n", "\n", "# Full similarity matrix\n", "im0 = axes[0, 0].imshow(sim_matrix, cmap='RdYlGn', vmin=-0.5, vmax=1.0, aspect='auto')\n", "axes[0, 0].set_title('Full Patch Similarity Matrix\\n[256×256]', fontweight='bold')\n", "axes[0, 0].set_xlabel('Patch index'); axes[0, 0].set_ylabel('Patch index')\n", "plt.colorbar(im0, ax=axes[0, 0], fraction=0.046)\n", "\n", "# Similarity maps for query patches\n", "for col, (qr, qc, label) in enumerate(query_patches):\n", " query_idx = qr * n_grid + qc\n", " sim_to_query = sim_matrix[query_idx].reshape(n_grid, n_grid)\n", " \n", " im = axes[0, col+1].imshow(sim_to_query, cmap='RdYlGn', vmin=-0.5, vmax=1.0,\n", " interpolation='bilinear')\n", " # Mark query patch\n", " axes[0, col+1].add_patch(plt.Rectangle((qc-0.5, qr-0.5), 1, 1,\n", " linewidth=3, edgecolor='black', facecolor='none'))\n", " axes[0, col+1].set_title(f'Similarity to {label}\\n(patch [{qr},{qc}])',\n", " fontweight='bold', fontsize=9)\n", " axes[0, col+1].axis('off')\n", " plt.colorbar(im, ax=axes[0, col+1], fraction=0.046)\n", "\n", "# Upsampled similarity overlays\n", "axes[1, 0].imshow(test_image)\n", "axes[1, 0].set_title('Original Image', fontweight='bold')\n", "axes[1, 0].axis('off')\n", "\n", "for col, (qr, qc, label) in enumerate(query_patches):\n", " query_idx = qr * n_grid + qc\n", " sim_to_query = sim_matrix[query_idx].reshape(n_grid, n_grid)\n", " \n", " # Normalize to [0,1]\n", " sim_norm = (sim_to_query - sim_to_query.min()) / (sim_to_query.max() - sim_to_query.min() + 1e-8)\n", " \n", " # Upsample\n", " sim_up = np.array(Image.fromarray((sim_norm * 255).astype(np.uint8)).resize(\n", " (cfg.image_size, cfg.image_size), Image.BILINEAR)) / 255.0\n", " \n", " heatmap = cm.RdYlGn(sim_up)[:, :, :3]\n", " img_np = np.array(test_image) / 255.0\n", " overlay = 0.5 * img_np + 0.5 * heatmap\n", " \n", " axes[1, col+1].imshow(overlay.clip(0, 1))\n", " # Mark query patch on overlay\n", " rect = plt.Rectangle((qc*cfg.patch_size, qr*cfg.patch_size),\n", " cfg.patch_size, cfg.patch_size,\n", " linewidth=3, edgecolor='black', facecolor='none')\n", " axes[1, col+1].add_patch(rect)\n", " axes[1, col+1].set_title(f'Similarity Overlay\\n{label}', fontweight='bold', fontsize=9)\n", " axes[1, col+1].axis('off')\n", "\n", "plt.suptitle('Patch Feature Similarity Maps\\n'\n", " 'Which patches are semantically similar to the query patch?',\n", " fontsize=14, fontweight='bold')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 8.5 Cross-Image Patch Correspondence" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ── Cross-image patch correspondence ──────────────────────────────────────────\n", "# Create two similar images (same structure, different colors)\n", "def make_image_v2(color1, color2, color3, color4, size=224):\n", " img = Image.new('RGB', (size, size), 'white')\n", " draw = ImageDraw.Draw(img)\n", " h = size // 2\n", " draw.rectangle([0, 0, h, h], fill=color1)\n", " draw.rectangle([h, 0, size, h], fill=color2)\n", " draw.rectangle([0, h, h, size], fill=color3)\n", " draw.rectangle([h, h, size, size], fill=color4)\n", " r = size // 14\n", " for cx, cy in [(h//2,h//2),(h+h//2,h//2),(h//2,h+h//2),(h+h//2,h+h//2)]:\n", " draw.ellipse([cx-r,cy-r,cx+r,cy+r], fill='white')\n", " return img\n", "\n", "img_a = make_image_v2((220,50,50), (50,100,220), (50,180,50), (220,200,50)) # original\n", "img_b = make_image_v2((50,50,180), (180,50,50), (200,180,50), (50,160,50)) # swapped colors\n", "\n", "feats_a = extract_patch_features(img_a) # [256, 768]\n", "feats_b = extract_patch_features(img_b) # [256, 768]\n", "\n", "# Normalize\n", "fa_norm = feats_a / (np.linalg.norm(feats_a, axis=1, keepdims=True) + 1e-8)\n", "fb_norm = feats_b / (np.linalg.norm(feats_b, axis=1, keepdims=True) + 1e-8)\n", "\n", "# Cross-image similarity\n", "cross_sim = fa_norm @ fb_norm.T # [256, 256]\n", "\n", "# For each patch in A, find best matching patch in B\n", "best_match_b = cross_sim.argmax(axis=1) # [256]\n", "best_match_sim = cross_sim.max(axis=1) # [256]\n", "\n", "fig, axes = plt.subplots(2, 4, figsize=(20, 10))\n", "\n", "axes[0, 0].imshow(img_a)\n", "axes[0, 0].set_title('Image A', fontweight='bold')\n", "axes[0, 0].axis('off')\n", "\n", "axes[0, 1].imshow(img_b)\n", "axes[0, 1].set_title('Image B', fontweight='bold')\n", "axes[0, 1].axis('off')\n", "\n", "# Cross-similarity matrix\n", "im2 = axes[0, 2].imshow(cross_sim, cmap='RdYlGn', vmin=-0.5, vmax=1.0, aspect='auto')\n", "axes[0, 2].set_title('Cross-Image Similarity\\n[256×256]', fontweight='bold')\n", "axes[0, 2].set_xlabel('Patch in B'); axes[0, 2].set_ylabel('Patch in A')\n", "plt.colorbar(im2, ax=axes[0, 2], fraction=0.046)\n", "\n", "# Best match similarity map\n", "best_sim_grid = best_match_sim.reshape(n_grid, n_grid)\n", "im3 = axes[0, 3].imshow(best_sim_grid, cmap='RdYlGn', vmin=0, vmax=1, interpolation='nearest')\n", "axes[0, 3].set_title('Best Match Similarity\\n(A→B, per patch)', fontweight='bold')\n", "axes[0, 3].axis('off')\n", "plt.colorbar(im3, ax=axes[0, 3], fraction=0.046)\n", "\n", "# Show correspondence for a few query patches\n", "query_patches_ab = [(n_grid//4, n_grid//4), (n_grid//4, 3*n_grid//4),\n", " (3*n_grid//4, n_grid//4), (3*n_grid//4, 3*n_grid//4)]\n", "\n", "for col, (qr, qc) in enumerate(query_patches_ab):\n", " query_idx = qr * n_grid + qc\n", " match_idx = best_match_b[query_idx]\n", " match_r, match_c = match_idx // n_grid, match_idx % n_grid\n", " sim_val = best_match_sim[query_idx]\n", " \n", " # Show A with query marked\n", " img_a_copy = np.array(img_a).copy()\n", " img_b_copy = np.array(img_b).copy()\n", " \n", " # Draw rectangles\n", " combined = np.zeros((cfg.image_size, cfg.image_size * 2 + 10, 3), dtype=np.uint8)\n", " combined[:, :cfg.image_size] = img_a_copy\n", " combined[:, cfg.image_size+10:] = img_b_copy\n", " \n", " combined_img = Image.fromarray(combined)\n", " draw = ImageDraw.Draw(combined_img)\n", " # Query in A\n", " x1a = qc * cfg.patch_size\n", " y1a = qr * cfg.patch_size\n", " draw.rectangle([x1a, y1a, x1a+cfg.patch_size, y1a+cfg.patch_size],\n", " outline=(255, 0, 0), width=3)\n", " # Match in B\n", " x1b = cfg.image_size + 10 + match_c * cfg.patch_size\n", " y1b = match_r * cfg.patch_size\n", " draw.rectangle([x1b, y1b, x1b+cfg.patch_size, y1b+cfg.patch_size],\n", " outline=(0, 255, 0), width=3)\n", " # Line connecting them\n", " draw.line([(x1a+cfg.patch_size//2, y1a+cfg.patch_size//2),\n", " (x1b+cfg.patch_size//2, y1b+cfg.patch_size//2)],\n", " fill=(255, 255, 0), width=2)\n", " \n", " axes[1, col].imshow(combined_img)\n", " axes[1, col].set_title(f'Query [{qr},{qc}] → Match [{match_r},{match_c}]\\n'\n", " f'Similarity: {sim_val:.3f}',\n", " fontweight='bold', fontsize=8)\n", " axes[1, col].axis('off')\n", "\n", "plt.suptitle('Cross-Image Patch Correspondence\\n'\n", " 'DINOv2 finds matching patches across different images',\n", " fontsize=14, fontweight='bold')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 8.6 Summary\n", "\n", "| Visualization | What it shows |\n", "|---------------|---------------|\n", "| Feature norm map | Which patches have strongest activations |\n", "| PCA RGB map | Semantic structure (colors = semantic regions) |\n", "| K-means clusters | Unsupervised segmentation |\n", "| Similarity map | Which patches are semantically similar |\n", "| Cross-image correspondence | Matching patches across images |\n", "\n", "**Key insight:** DINOv2 patch features encode **semantic content**, not just pixel values. \n", "Patches with similar content (same color region, same object part) have similar features.\n", "\n", "---\n", "**Next:** [`09_cls_token_analysis.ipynb`](09_cls_token_analysis.ipynb)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "name": "python", "version": "3.10.0" } }, "nbformat": 4, "nbformat_minor": 5 }