commit 34a885bcf8f3141b9990821e28f23312985103ac Author: minsung Date: Wed Jul 15 16:11:59 2026 +0900 Initial publish: SamGeo3 multi-prompt segmentation lab. Scripts, prompt JSON tiers, usage docs, and README. Input images (data/) and segmentation outputs (output/) are gitignored. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1d95d5a --- /dev/null +++ b/.gitignore @@ -0,0 +1,45 @@ +.venv/ +__pycache__/ +*.py[cod] +*.egg-info/ +.ipynb_checkpoints/ +.DS_Store +Thumbs.db + +# Local secrets / HF +.env +.hf_token + +# ---- Input / output data (publish code only) ---- +# Keep folder structure via .gitkeep; ignore all payloads. +data/* +!data/.gitkeep +output/* +!output/.gitkeep + +# Large model / geospatial / mask artifacts (anywhere) +*.pt +*.pth +*.ckpt +*.npy +*.tif +*.tiff +*.jp2 +*.gpkg +*.shp +*.dbf +*.shx +*.prj +*.las +*.laz + +# Accidental root-level media dumps +/*.jpg +/*.jpeg +/*.JPG +/*.JPEG +/*.png +/*.PNG +/*.webp +/*.mp4 +/*.mov diff --git a/README.md b/README.md new file mode 100644 index 0000000..9243ece --- /dev/null +++ b/README.md @@ -0,0 +1,180 @@ +# samgeo3-lab + +독립 SamGeo3 (SAM 3 / SAM 3.1) 세그멘테이션 실험 프로젝트. + +- **패키지**: [segment-geospatial](https://github.com/opengeos/segment-geospatial) (`SamGeo3`) +- **문서**: https://samgeo.gishub.org +- **범위**: 설치 검증 + 단일 이미지 텍스트 프롬프트 세그멘테이션 +- **비범위**: railway-client / sam31server 통합, 철도 후처리, REST API + +기존 SAM 서버 env/코드와 **공유하지 않습니다**. 이 폴더의 `.venv`만 사용하세요. + +## 가정 (이 머신 기준) + +| 항목 | 상태 | +|------|------| +| GPU | NVIDIA RTX 3060 (드라이버 CUDA 13.x 가능) | +| conda / pixi | 미설치 → **`uv` + Python 3.12 venv** 사용 | +| HF 토큰 | `~/.cache/huggingface/token` 존재 | +| 체크포인트 | `facebook/sam3.1` HF 캐시 이미 존재 (~3.3GB) | + +## 폴더 구조 + +``` +samgeo3-lab/ + README.md + requirements.txt + pyproject.toml + scripts/ + check_install.py # 설치/GPU/import 확인 (+ 옵션: 모델 로드) + text_segment.py # 단일 이미지 텍스트 세그멘테이션 + multi_prompt_segment.py # 멀티 프롬프트 일괄 세그 (모델 1회 로드) + prompts/ + dji_20260306_0016.json # Grok+Gemini 병합 multi-object 프롬프트 + README.md + docs/usage.html + data/ # 입력 이미지 + output/ # 마스크/시각화 결과 + .venv/ # 로컬 가상환경 (gitignore) +``` + +## 설치 (Windows + GPU) + +PowerShell, 프로젝트 루트에서: + +```powershell +cd D:\MYCLAUDE_PROJECT\samgeo3-lab + +# 1) 독립 venv (Python 3.12) +uv venv --python 3.12 .venv +.\.venv\Scripts\Activate.ps1 + +# 2) CUDA PyTorch 먼저 (CPU wheel 덮어쓰기 방지) +uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121 + +# 3) SamGeo3 extras only (api / all 설치 금지) +uv pip install "segment-geospatial[samgeo3]>=1.4.1" matplotlib + +# 4) Windows: sam3 import에 필요 (pip extra가 빠질 수 있음) +uv pip install "triton-windows>=3.3.0.post19" + +# 5) (선택) 노트북 +uv pip install ipykernel jupyterlab +``` + +### 대안: pixi (공식 권장) + +conda/pixi를 쓸 수 있으면 공식 문서 권장 경로가 더 안정적입니다. +https://samgeo.gishub.org/installation/#install-with-pixi-recommended + +### Hugging Face (SAM 3 / 3.1) + +1. https://huggingface.co/facebook/sam3.1 접근 승인 +2. 인증: `hf auth login` (또는 기존 토큰 파일 유지) +3. (선택) 로컬 체크포인트만 쓰려면: + +```powershell +$env:SAM3_CHECKPOINT_PATH = "C:\path\to\sam3.1_multiplex.pt" +``` + +## 실행 + +```powershell +cd D:\MYCLAUDE_PROJECT\samgeo3-lab +.\.venv\Scripts\Activate.ps1 + +# A) 패키지/GPU 확인 +python scripts/check_install.py + +# B) 모델 로드까지 확인 (캐시/다운로드) +python scripts/check_install.py --load-model + +# C) 샘플 이미지 텍스트 세그멘테이션 +python scripts/text_segment.py +# 동의어 (기본 confidence=0.3 — sam3.1 score가 0.5 미만인 경우 많음): +python scripts/text_segment.py --prompt person --confidence 0.3 + +# D) 자체 이미지 +python scripts/text_segment.py --image data\my.jpg --prompt building --min-size 100 + +# E) 멀티 프롬프트 (Grok+Gemini 병합 세트, 모델 1회 로드) +python scripts/multi_prompt_segment.py --list-only --tier compact +python scripts/multi_prompt_segment.py --tier compact --confidence 0.3 +python scripts/multi_prompt_segment.py --tier A_high +python scripts/multi_prompt_segment.py --tier all --confidence 0.25 +# DJI 드론 정사 기본 이미지: data/DJI_20260306100802_0016.JPG +# 프롬프트 JSON: prompts/dji_20260306_0016.json + +# F) 누락 보강 (Gemini gap) — 같은 output 폴더에 추가 +python scripts\multi_prompt_segment.py ` + --tier gap --confidence 0.25 ` + --output-dir output\dji_0016_compact + +# G) 멀티 결과 한 장으로 합치기 (overlay + grid + HTML) +python scripts\merge_multi_results.py ` + --result-dir output\dji_0016_compact ` + --image data\DJI_20260306100802_0016.JPG +# → output\dji_0016_compact\merged\index.html (디스크 마스크 전부 병합) + +# H) 예시: DJI_0100 전체 파이프라인 (0044 프롬프트 → compact → gap → gap2_core → merge) +$img = "data\DJI_20260306101434_0100.JPG" +$out = "output\dji_0100_compact" +$json = "prompts\dji_20260306_0044.json" + +python scripts\multi_prompt_segment.py --prompts-json $json --list-only --tier compact + +python scripts\multi_prompt_segment.py ` + --prompts-json $json --image $img --tier compact --confidence 0.3 --output-dir $out +# (선택) gap / gap2_core 보강 — 같은 $out 에 추가 +python scripts\multi_prompt_segment.py ` + --prompts-json $json --image $img --tier gap --confidence 0.25 --output-dir $out +python scripts\multi_prompt_segment.py ` + --prompts-json $json --image $img --tier gap2_core --confidence 0.22 --output-dir $out + +python scripts\merge_multi_results.py --result-dir $out --image $img --max-side 2048 +start "$out\merged\index.html" +``` + +상세 주석 버전은 `prompts/README.md` §「DJI_0100 전체 파이프라인」을 보세요. + +결과 파일은 `output/` 아래에 저장됩니다. + +| 파일 | 내용 | +|------|------| +| `*_mask.png` | 인스턴스별 unique mask | +| `*_scores.npy` | 객체별 confidence 점수 | +| `*_ann.png` | 오버레이 시각화 | + +**참고**: 모델 로드 시 `missing_keys` 경고가 날 수 있음 (`sam3` 패키지 vs `sam3.1` 체크포인트 일부 키 불일치). 추론은 동작하나 score가 낮을 수 있어 `--confidence` 튜닝이 필요합니다. + +## SamGeo3 빠른 참고 + +```python +from samgeo import SamGeo3 + +sam = SamGeo3( + backend="meta", # sam3.1 은 meta 전용 + model_id="facebook/sam3.1", + confidence_threshold=0.5, + resolution=1008, +) +sam.set_image("data/test_image.jpg") +sam.generate_masks("person") +sam.save_masks("output/mask.png") +``` + +- `backend="meta"`: `facebook/sam3`, `facebook/sam3.1`, 배치·인터랙티브 지원 +- `backend="transformers"`: `facebook/sam3` 만 + +## 검증 순서 + +1. `check_install.py` — CUDA + imports +2. `check_install.py --load-model` — 가중치 로드 +3. `text_segment.py` — 마스크 저장 확인 +4. `confidence` / `min_size` 튜닝 후 실데이터 실험 + +## 참고 + +- 업스트림: https://github.com/opengeos/segment-geospatial +- API 문서: https://samgeo.gishub.org/samgeo3/ +- 라이선스(업스트림): MIT diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/dji_0044_prompts.html b/docs/dji_0044_prompts.html new file mode 100644 index 0000000..96170cf --- /dev/null +++ b/docs/dji_0044_prompts.html @@ -0,0 +1,251 @@ + + + + + + DJI_0044 = 0016 확장 — 철로+도로 + + + +
+
+

DJI_0044 = 이미지 1번(0016) 확장

+

+ 별도 체계가 아닙니다. 0016 프롬프트·tier·gap/gap2 워크플로를 상속하고, + 이 장면(철로+도로+열차+야적)에만 필요한 객체를 scene_delta 로 덧붙입니다. +

+
+ extends 0016 + same pipeline + scene_delta only + SAM 3.1 meta +
+
+ + + +
+

1. 확장 모델

+
+ ① DJI_0016 base
compact / gap / gap2 + → + ② scene_delta
열차·도로·야적 + → + ③ DJI_0044 compact
base + delta + → + ④ merge +
+ + + + + + + + + + + + + + + + + + + + + + +
개념설명
베이스prompts/dji_20260306_0016.json — 철로·산업 orthophoto에서 검증한 전체 세트
확장 파일prompts/dji_20260306_0044.json — "extends": "…0016.json"
상속A~E tier, compact, gap, improved, gap2, gap2_core 전부 사용 가능
추가scene_delta.prompts + 필요 시 tier별 덧붙임
스크립트multi_prompt_segment.py 가 extends를 자동 병합
+
+ 원칙 + 새 이미지가 들어와도 “새 체계”를 만들지 않고, + 1번에서 쌓은 프롬프트 자산 + 장면 delta 로만 확장합니다. +
+
+ +
+

2. scene_delta (0044에서 추가되는 객체)

+

Gemini 분석 + 비전 확인. 0016에 없거나 이 장면에서 핵심인 항목.

+ + + + + + + + + + + + + + + + + + + + +
그룹프롬프트
열차train, freight car, container car, locomotive, passenger train…
도로highway, pedestrian crossing, crosswalk, road marking
야적pipe pile, pipes, bag pile, wooden pile, wood stack, tarp, gravel…
기타chain-link fence, sleeper, trackbed, catenary, red roof…
+

+ 베이스에 이미 있는 것(railway track, building, road, + car, blue tarp, material pile 등)은 + 다시 정의하지 않고 상속합니다. +

+
+ +
+

3. 실행 순서 (0016과 동일 파이프라인)

+
    +
  1. --tier compact — 0016 compact + scene_delta
    2. +
  2. --tier gap — 0016 누락 보강 세트 (상속)
    3. +
  3. --tier gap2 또는 gap2_core — 합본 빈 영역 보강 (상속)
    4. +
  4. --tier delta — 0044 추가분만 (빠른 재실험)
    5. +
  5. merge_multi_results.py
    6. +
+
+ tier 의미 + gap / gap2 는 0044 전용이 아니라 + 0016에서 만든 보강 세트 그대로입니다. 같은 철로 orthophoto 계열에 재사용. +
+
+ +
+

4. 커맨드

+ 
cd D:\MYCLAUDE_PROJECT\samgeo3-lab
+.\.venv\Scripts\Activate.ps1
+
+# 상속 결과 확인 (compact = 0016 + delta)
+python scripts\multi_prompt_segment.py `
+  --prompts-json prompts\dji_20260306_0044.json `
+  --list-only --tier compact
+
+# 이 장면 추가분만
+python scripts\multi_prompt_segment.py `
+  --prompts-json prompts\dji_20260306_0044.json `
+  --list-only --tier delta
+
+# 본 실행 (1번과 동일 패턴)
+python scripts\multi_prompt_segment.py `
+  --prompts-json prompts\dji_20260306_0044.json `
+  --image data\DJI_20260306100928_0044.JPG `
+  --tier compact --confidence 0.3 `
+  --output-dir output\dji_0044_compact
+
+python scripts\multi_prompt_segment.py `
+  --prompts-json prompts\dji_20260306_0044.json `
+  --tier gap --confidence 0.25 `
+  --output-dir output\dji_0044_compact
+
+python scripts\merge_multi_results.py `
+  --result-dir output\dji_0044_compact `
+  --image data\DJI_20260306100928_0044.JPG
+
+ +
+

5. 파일

+ + + + + + + + + + + + + + + + + + +
경로역할
prompts/dji_20260306_0016.json베이스 (이미지 1)
prompts/dji_20260306_0044.jsonextends 0016 + scene_delta
docs/dji_gap_fill.html0016 gap 리포트
output/dji_0044_compact/merged/0044 합본
+

← 0016 갭 필 리포트 · 일반 사용법

+
+ +
+ samgeo3-lab · 0044 extends 0016 · 별도 체계 아님
+ docs/dji_0044_prompts.html +
+
+ + diff --git a/docs/dji_gap_fill.html b/docs/dji_gap_fill.html new file mode 100644 index 0000000..3b06d2e --- /dev/null +++ b/docs/dji_gap_fill.html @@ -0,0 +1,534 @@ + + + + + + DJI 정사 멀티 프롬프트 · 갭 필 리포트 — samgeo3-lab + + + +
+
+

DJI 정사 · 멀티 프롬프트 갭 필 리포트

+

+ SamGeo3 / SAM 3.1 텍스트 세그 실험 — + compact → gap → gap2 단계로 빈 영역을 메운 과정과 결과 요약. +

+
+ facebook/sam3.1 + backend=meta + DJI_20260306100802_0016 + 56 classes merged + samgeo3-lab +
+
+ + + +
+

1. 개요 · 파이프라인

+
+
compact
1차 19 프롬프트 → ~18 마스크
+
gap
Gemini 누락 보강 → road, forest…
+
gap2
합본 빈 영역 44 프롬프트 → 28 성공
+
56
최종 merge 클래스 수
+
+ 
# 전체 흐름
+multi_prompt_segment.py --tier compact   # 기본 객체
+multi_prompt_segment.py --tier gap       # 1차 누락
+multi_prompt_segment.py --tier gap2      # 합본 빈 영역
+merge_multi_results.py                   # overlay + legend + HTML
+

+ 설정: prompts/dji_20260306_0016.json · + 사용법 일반: docs/usage.html +

+
+ +
+

2. 대상 이미지

+ + + + + + + + + + + + + + + + + + + + + + +
항목값
파일data/DJI_20260306100802_0016.JPG
해상도8192 × 5460 (드론 정사에 가까운 수직뷰)
장면복선 철로 · 창고/주차장 · 태양광 지붕 · 수목 · 야적장
모델facebook/sam3.1 · backend=meta
confidencecompact 0.3 · gap 0.25 · gap2 0.22
+
+ +
+

3. 합본에서 보인 빈 영역

+

+ 1차 merge(combined_with_legend.png) 기준으로 + 마스크 색이 없고 원본 톤이 남는 구간을 재검토했습니다. +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
구역추정 내용1차에서 부족한 이유
선로 상·하 갈색 띠성토·제방·마른 초지railway track 밴드만, 바깥 지면 미채움
검은 타프 주변흙더미·절토black tarp만, 주변 토사 약함
시설 사이 공터콘크리트·야적 바닥parking lot / road와 다른 표면
하측 야적녹생 덮개·시트색 지붕 프롬프트와 다른 tarp/cover
소형 구조shed · 부스 · 차양building에 안 묶임
지붕 잔여금속 / 골판 / 회색white·green·blue roof만으로는 부족
선형 세부도상 · 전선정사 + 텍스트 그라운딩 한계
+
+ +
+

4. 프롬프트 tier

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
tier설명권장 conf
compact1차 실험 19개 (철로·건물·차량·타프…)0.30
gapGemini 누락 보강 (road, dense forest, covered truck…)0.25
gap2합본 빈 영역 전체 44개0.22
gap2_coregap2 중 실측 성공만 (~18개, 재실행용)0.22
improvedcompact + gap 병합0.28
allA~E 전체 (중복 제거)0.25~0.30
+
+ +
+

5. gap (1차 누락 보강)

+

Gemini 재검토 + 실측 성공 동의어

+
+ road + asphalt pavement + gravel path + dense forest + covered truck + cargo vehicle + debris + rubbish + light pole + scrap pile + street light + lamp post + drainage ditch + industrial waste +
+
+ 팁 + 가로등: light pole @ 0.2 가 유효. + 배수로·전선은 정사에서 거의 실패. + rubbish / scrap pile 은 노이즈 과다 가능. +
+
+ +
+

6. gap2 (합본 재검토 보강)

+

타겟 갭 (JSON notes)

+
    +
  • 선로 상·하 갈색 성토/제방·마른 초지
    • +
  • 검은 타프 주변 흙더미·절토면
    • +
  • 주차장·창고 사이 콘크리트·공터
    • +
  • 하측 야적 녹생 방수시트/덮개
    • +
  • 소형 shed · 담장 · 차양
    • +
  • 금속/골판/회색 지붕 잔여
    • +
  • 건물 벽면 · 장벽
    • +
+ +

성공 프롬프트 28/44 (conf=0.22)

+
+ dirt embankment + dirt pile + dry grass + bushes + grass + vegetation + brown field + open ground + concrete + concrete pad + concrete pavement + industrial yard + paved yard + green tarp + green plastic sheet + plastic cover + shed + small shed + metal roof + corrugated roof + gray roof + wall + building wall + fence wall + barrier + canopy + awning + sidewalk +
+ +

실패 16

+
+ embankment + railway embankment + soil pile + earth mound + scrub + dormant vegetation + booth + kiosk + retaining wall + ballast + track bed + railway ballast + power line + overhead wire + shadow + walkway +
+ +

gap2_core (재실행 권장)

+ 
"dirt embankment", "dirt pile", "dry grass", "bushes", "vegetation",
+"brown field", "open ground", "concrete", "industrial yard",
+"green tarp", "plastic cover", "shed", "metal roof", "corrugated roof",
+"gray roof", "building wall", "barrier", "awning"
+
+ +
+

7. 실측 결과 요약

+ + + + + + + + + + + + + + + + +
단계merge 클래스비고
compact + gap~28road / dense forest / covered truck 등 추가
+ gap256성토·초지·콘크리트·shed·금속지붕 등
+
+ 과대 마스크 주의 + industrial yard, open ground, dirt embankment + 픽셀 면적이 커서 다른 클래스를 덮을 수 있습니다. + 필요 시 해당 *_mask.png 를 제외하고 다시 merge 하세요. +
+
+ 빈 영역 개선에 특히 유효했던 프롬프트 + dirt embankment · dry grass · vegetation · concrete · green tarp · + shed · metal/corrugated roof · building wall · awning +
+
+ +
+

8. 실행 커맨드

+ 
cd D:\MYCLAUDE_PROJECT\samgeo3-lab
+.\.venv\Scripts\Activate.ps1
+
+# 1차
+python scripts\multi_prompt_segment.py `
+  --image "data\DJI_20260306100802_0016.JPG" `
+  --tier compact --confidence 0.3 `
+  --output-dir "output\dji_0016_compact"
+
+# Gemini 누락 보강 (같은 폴더에 마스크 추가)
+python scripts\multi_prompt_segment.py `
+  --tier gap --confidence 0.25 `
+  --output-dir "output\dji_0016_compact"
+
+# 합본 빈 영역 보강
+python scripts\multi_prompt_segment.py `
+  --tier gap2 --confidence 0.22 `
+  --output-dir "output\dji_0016_compact"
+
+# 또는 성공 프롬프트만
+python scripts\multi_prompt_segment.py `
+  --tier gap2_core --confidence 0.22 `
+  --output-dir "output\dji_0016_compact"
+
+# 한 장으로 합치기
+python scripts\merge_multi_results.py `
+  --result-dir "output\dji_0016_compact" `
+  --image "data\DJI_20260306100802_0016.JPG" `
+  --max-side 2560
+
+ +
+

9. 결과 파일

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
경로내용
output/dji_0016_compact/*_mask.png프롬프트별 인스턴스 마스크
output/dji_0016_compact/*_ann.png개별 오버레이 (대용량 가능)
output/dji_0016_compact/merged/combined_with_legend.png전 클래스 합본 + 범례
output/dji_0016_compact/merged/per_class_grid.png클래스별 썸네일 격자
output/dji_0016_compact/merged/index.html브라우저 뷰 (결과 이미지)
output/dji_0016_compact/merged/merge_summary.json클래스·픽셀 통계
prompts/dji_20260306_0016.jsontier · gap · gap2 · gap2_core 정의
+

+ 결과 미리보기 HTML: + merged/index.html + (상대 경로; 로컬에서 파일 열기) +

+
+ +
+

10. 한계 · 다음 단계

+
    +
  • SAM 3.1 텍스트 세그는 개념 단위 인스턴스이지 픽셀 토지피복 분류기가 아님.
    • +
  • 도상·전선·그림자 등 가늘거나 추상적인 개념은 텍스트만으로 약함.
    • +
  • 클래스 간 겹침이 큼 → 우선순위 합성 / 배타 마스크 후처리 권장.
    • +
  • 작은 시설물은 점·박스 인터랙티브(enable_inst_interactivity)가 유리.
    • +
  • 초대형 정사는 generate_masks_tiled 로 VRAM·경계 품질 개선 가능.
    • +
+
+ 권장 운영 + compact → gap → gap2_core → merge → 과대 클래스 수동 제외 → 재 merge. + confidence는 0.22~0.35 구간에서 장면별로 조정. +
+
+ +
+ samgeo3-lab · DJI gap-fill report · 2026-07-15
+ 파일: docs/dji_gap_fill.html + · 일반 사용법: usage.html +
+
+ + diff --git a/docs/usage.html b/docs/usage.html new file mode 100644 index 0000000..b99dca2 --- /dev/null +++ b/docs/usage.html @@ -0,0 +1,763 @@ + + + + + + samgeo3-lab 사용법 — SamGeo3 / SAM 3.1 + + + +
+
+

samgeo3-lab 사용법

+

+ segment-geospatial의 SamGeo3 백엔드로 + SAM 3 / SAM 3.1 텍스트 프롬프트 세그멘테이션을 실험하는 + 독립 프로젝트 가이드입니다. +

+
+ Python 3.12 + CUDA / RTX GPU + backend=meta + facebook/sam3.1 + Windows +
+
+ + + +
+

1. 개요 · 범위

+

+ 프로젝트 경로: D:\MYCLAUDE_PROJECT\samgeo3-lab +

+ + + + + + + + + + + + + + + + + + + + + + +
항목내용
패키지segment-geospatial (PyPI) — 클래스 SamGeo3
하는 일설치 검증, 단일 이미지 텍스트 세그멘테이션, 마스크/시각화 저장
하지 않는 일railway-client / sam31server 통합, 철도 후처리, REST API, AnyLabeling/YOLO 파이프라인
환경 원칙이 폴더의 .venv만 사용. 다른 SAM 서버 env와 섞지 않음
+
+ 이미 구성된 경우 + 설치 단계는 건너뛰고 §5 실행 커맨드부터 진행하면 됩니다. +
+
+ +
+

2. 폴더 구조

+ 
samgeo3-lab/
+  README.md
+  docs/usage.html          ← 이 문서
+  requirements.txt
+  pyproject.toml
+  scripts/
+    check_install.py       # GPU / import / (옵션) 모델 로드 확인
+    text_segment.py        # 단일 이미지 텍스트 세그멘테이션
+  data/                    # 입력 이미지
+  output/                  # 마스크 · 점수 · 오버레이 결과
+  .venv/                   # 독립 가상환경
+
+ +
+

3. 설치 (Windows + GPU)

+

PowerShell에서 프로젝트 루트로 이동한 뒤 순서대로 실행합니다.

+ 
cd D:\MYCLAUDE_PROJECT\samgeo3-lab
+
+# 1) 독립 venv (Python 3.12)
+uv venv --python 3.12 .venv
+.\.venv\Scripts\Activate.ps1
+
+# 2) CUDA PyTorch 먼저 (CPU wheel 덮어쓰기 방지)
+uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121
+
+# 3) SamGeo3 extras only (api / all 설치 금지)
+uv pip install "segment-geospatial[samgeo3]>=1.4.1" matplotlib
+
+# 4) Windows: sam3 import에 필요
+uv pip install "triton-windows>=3.3.0.post19"
+
+# 5) (선택) 노트북
+uv pip install ipykernel jupyterlab
+ +
+ 주의 + triton-windows가 없으면 import sam3 가 실패합니다. + segment-geospatial[samgeo3] 설치 후에도 반드시 확인하세요. +
+ +

대안: pixi (공식 권장)

+

+ conda/pixi 환경이 가능하면 PyTorch·CUDA·SAM3 의존성 해결에 유리합니다.
+ + samgeo.gishub.org — Install with pixi + +

+
+ +
+

4. Hugging Face · 체크포인트

+
    +
  1. + 모델 접근 승인
    + facebook/sam3.1 + (및 필요 시 sam3) 폼 제출 후 승인 +
    2. +
  2. + 인증
    + hf auth login 또는 기존 토큰 파일 + ~/.cache/huggingface/token 유지 +
    3. +
  3. + (선택) 로컬 체크포인트 + 
    $env:SAM3_CHECKPOINT_PATH = "C:\path\to\sam3.1_multiplex.pt"
    + 스크립트에서는 --checkpoint 로도 지정 가능합니다. +
    4. +
+
+ 백엔드 제약 +
    +
  • backend="meta" — facebook/sam3, facebook/sam3.1 (배치·인터랙티브 풍부)
    • +
  • backend="transformers" — facebook/sam3 만. sam3.1은 meta 전용
    • +
+
+
+ +
+

5. 실행 커맨드

+

매번 가상환경을 활성화한 뒤 실행합니다.

+ 
cd D:\MYCLAUDE_PROJECT\samgeo3-lab
+.\.venv\Scripts\Activate.ps1
+
+# A) 패키지 / GPU / import 확인
+python scripts\check_install.py
+
+# B) 모델 로드까지 확인 (HF 캐시 또는 다운로드)
+python scripts\check_install.py --load-model
+
+# C) 샘플 이미지 텍스트 세그멘테이션 (기본 prompt=person, confidence=0.3)
+python scripts\text_segment.py
+python scripts\text_segment.py --prompt person --confidence 0.3
+
+# D) 자체 이미지
+python scripts\text_segment.py --image data\my.jpg --prompt building --min-size 100
+ +

권장 검증 순서

+
    +
  1. check_install.py — CUDA + imports
    2. +
  2. check_install.py --load-model — 가중치 로드
    3. +
  3. text_segment.py — 마스크 저장 확인
    4. +
  4. --confidence / --min-size 튜닝 후 실데이터 실험
    5. +
+
+ +
+

6. text_segment.py 옵션

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
옵션기본값설명
--imagedata/test_image.jpg입력 이미지 경로 (없으면 샘플 자동 다운로드)
--promptperson텍스트 프롬프트 (예: building, tree, car)
--model-idfacebook/sam3.1HF 모델 ID
--backendmetameta | transformers
--confidence0.3confidence threshold. 0.5면 마스크 0개일 수 있음
--mask-threshold0.5transformers 후처리용 마스크 threshold
--resolution1008meta 입력 해상도
--min-size / --max-size0 / 없음마스크 픽셀 면적 필터
--device자동cuda | cpu
--checkpoint환경변수 또는 없음로컬 .pt 경로
--output-diroutput/결과 저장 디렉터리
--no-vizoff오버레이 PNG 생략 (마스크는 저장)
+ +

check_install.py 옵션

+ 
python scripts\check_install.py
+python scripts\check_install.py --load-model
+python scripts\check_install.py --load-model --model-id facebook/sam3.1 --device cuda
+
+ +
+

6b. 멀티 프롬프트 (Grok + Gemini 병합)

+

+ 드론 정사 장면 분석 결과를 합친 프롬프트 JSON과 일괄 실행 스크립트입니다. + 모델·이미지를 1회만 로드한 뒤 프롬프트를 순차 + generate_masks 합니다. +

+ + + + + + + + + + + + + + + + + + +
경로설명
prompts/dji_20260306_0016.jsontier A~D + compact, 장면 노트(KO), 권장 confidence
scripts/multi_prompt_segment.py멀티 프롬프트 러너 + summary.json
data/DJI_20260306100802_0016.JPG기본 입력 (없으면 sample 경로 fallback)
+ 
python scripts\multi_prompt_segment.py --list-only --tier compact
+python scripts\multi_prompt_segment.py --tier compact --confidence 0.3
+python scripts\multi_prompt_segment.py --tier A_high
+python scripts\multi_prompt_segment.py --tier all --confidence 0.25
+python scripts\multi_prompt_segment.py --prompts "railway track,building,solar panel"
+

+ tier: compact | gap | gap2 | gap2_core | + improved | all | A_high | + B_facility | C_detail | D_rail_domain +

+

결과 디렉터리: output/<이미지stem>_<tier>/ — 프롬프트별 mask/ann + summary.json

+ +

DJI 정사 — 단일 프롬프트 예시 (실제 경로)

+ 
cd D:\MYCLAUDE_PROJECT\samgeo3-lab
+.\.venv\Scripts\Activate.ps1
+
+$img = "data\DJI_20260306100802_0016.JPG"
+$out = "output\dji_0016_analysis"
+
+python scripts\text_segment.py --image $img --prompt "solar panel" --confidence 0.3 --output-dir $out
+python scripts\text_segment.py --image $img --prompt "building" --confidence 0.3 --output-dir $out
+python scripts\text_segment.py --image $img --prompt "railway track" --confidence 0.3 --output-dir $out
+ +

기대 난이도 (정사 드론)

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
프롬프트기대튜닝 팁
building, solar panel, tree, car, truck높음기본 객체. 낮은 confidence에서도 비교적 잘 잡힘
railway track, fence, parking lot, blue tarp중~높음선형·구획. context에 따라 변동
utility pole, container, material pile중작거나 밀집. --min-size 권장
rail, ballast, railroad sleeper, catenary낮~중도메인·가는 객체. confidence 튜닝 필수
+
+ 권장 + confidence 0.25~0.35 에서 시작. 작은 폴·미세 자재는 --min-size로 노이즈 필터. +
+ +

결과 합치기 (merge)

+

멀티 프롬프트 실행 후 클래스별 마스크를 한 장 오버레이 + 격자 + HTML로 합칩니다.

+ 
python scripts\merge_multi_results.py `
+  --result-dir "output\dji_0016_compact" `
+  --image "data\DJI_20260306100802_0016.JPG" `
+  --max-side 2560
+

출력: output\dji_0016_compact\merged\

+
    +
  • combined_with_legend.png — 전 클래스 오버레이 + 범례
    • +
  • per_class_grid.png — 클래스별 썸네일
    • +
  • index.html — 브라우저 뷰
    • +
+
+ +
+

6c. 예시: DJI_0100 전체 파이프라인

+

+ 0044 프롬프트 JSON(0016 상속 + 도로/열차 delta)을 + DJI_…_0100.JPG에 적용하는 end-to-end 예입니다. + 권장 순서: compact → (선택) gap → (선택) gap2_core → merge → HTML. +

+

+ confidence: compact 0.3 · gap 0.25 · gap2_core 0.22. + 같은 출력 폴더에 단계별로 마스크를 쌓은 뒤 merge하면 한 HTML에서 전부 볼 수 있습니다. +

+ 
cd D:\MYCLAUDE_PROJECT\samgeo3-lab
+.\.venv\Scripts\Activate.ps1
+
+$img = "data\DJI_20260306101434_0100.JPG"
+$out = "output\dji_0100_compact"
+$json = "prompts\dji_20260306_0044.json"
+
+# (선택) 프롬프트 목록 확인
+python scripts\multi_prompt_segment.py --prompts-json $json --list-only --tier compact
+
+# 1) 세그 — compact (0016 상속 + 도로/열차 delta)
+python scripts\multi_prompt_segment.py `
+  --prompts-json $json `
+  --image $img `
+  --tier compact `
+  --confidence 0.3 `
+  --output-dir $out
+
+# 2) (선택) gap 보강
+python scripts\multi_prompt_segment.py `
+  --prompts-json $json `
+  --image $img `
+  --tier gap `
+  --confidence 0.25 `
+  --output-dir $out
+
+# 3) (선택) gap2_core 보강
+python scripts\multi_prompt_segment.py `
+  --prompts-json $json `
+  --image $img `
+  --tier gap2_core `
+  --confidence 0.22 `
+  --output-dir $out
+
+# 4) 합치기 + interactive index.html
+python scripts\merge_multi_results.py `
+  --result-dir $out `
+  --image $img `
+  --max-side 2048
+
+# 5) 결과 열기
+start "$out\merged\index.html"
+
+ 결과 + 브라우저에서 output\dji_0100_compact\merged\index.html 을 열면 + 오버레이·클래스별 격자·범례를 한눈에 볼 수 있습니다. +
+
+ +
+

7. 출력 파일

+

결과는 output/ 아래에 저장됩니다. 파일명 패턴: {이미지stem}_{prompt}_*

+ + + + + + + + + + + + + + + + + + +
파일내용
*_mask.png인스턴스별 unique mask (객체마다 다른 픽셀 값)
*_scores.npy객체별 confidence 점수 (NumPy 배열). float PNG는 저장 불가
*_ann.png원본 + 마스크/박스/점수 오버레이 시각화
+

예: test_image_person_mask.png, test_image_person_ann.png

+
+ +
+

8. Python API 빠른 참고

+ 
from samgeo import SamGeo3
+
+sam = SamGeo3(
+    backend="meta",                 # sam3.1 은 meta 전용
+    model_id="facebook/sam3.1",
+    confidence_threshold=0.3,       # 환경에 맞게 튜닝
+    resolution=1008,
+    enable_segmentation=True,
+    enable_inst_interactivity=False,
+)
+sam.set_image("data/test_image.jpg")
+sam.generate_masks("person")        # 텍스트 프롬프트
+sam.save_masks("output/mask.png")
+sam.show_anns(output="output/ann.png")
+ +

포인트 / 박스 인터랙티브 (참고)

+ 
import numpy as np
+from samgeo import SamGeo3
+
+sam = SamGeo3(backend="meta", enable_inst_interactivity=True)
+sam.set_image("image.jpg")
+masks, scores, logits = sam.predict_inst(
+    point_coords=np.array([[520, 375]]),
+    point_labels=np.array([1]),   # 1=foreground, 0=background
+)
+ +

배치 (meta만)

+ 
sam = SamGeo3(backend="meta")
+sam.set_image_batch(["a.jpg", "b.jpg"])
+sam.generate_masks_batch("tree")
+sam.save_masks_batch("output/", prefix="tree_mask")
+
+ +
+

9. 튜닝 · 트러블슈팅

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
증상원인 / 대응
No module named 'triton'uv pip install "triton-windows>=3.3.0.post19"
No objects found + --confidence 낮추기 (예: 0.3 → 0.2), 프롬프트 변경, + 명확한 객체가 보이는 이미지 사용 +
로드 시 missing_keys 경고 + sam3 패키지와 sam3.1 체크포인트 일부 키 불일치. + 추론은 동작할 수 있으나 score가 낮을 수 있음 → confidence 튜닝 +
CUDA not available + CUDA torch 재설치, nvidia-smi 확인. + SAM3 meta는 GPU 권장 (CPU만으로는 제한) +
HF 접근 거부 / 다운로드 실패모델 게이트 승인 + hf auth login
작은 객체 과다 검출--min-size 로 픽셀 면적 필터
GeoTIFF 멀티밴드API: set_image(path, bands=[4,3,2]) (1-based band index)
+
+ confidence 기본값 0.3 이유 + 이 환경에서 sam3.1 meta score가 대략 0.33–0.35 부근으로 나오는 경우가 있어, + 기본 0.5를 쓰면 마스크가 0개일 수 있습니다. 실데이터에서 점수 분포를 보고 조정하세요. +
+
+ + + +
+ samgeo3-lab · 독립 SamGeo3 실험 환경 · 생성일 2026-07-15
+ 파일 위치: docs/usage.html +
+
+ + diff --git a/output/.gitkeep b/output/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/prompts/README.md b/prompts/README.md new file mode 100644 index 0000000..c167135 --- /dev/null +++ b/prompts/README.md @@ -0,0 +1,212 @@ +# Multi-object prompts (Grok + Gemini) + +비전 분석 결과를 합친 SAM 3.1 / SamGeo3 텍스트 프롬프트 세트입니다. + +| 파일 | 설명 | +|------|------| +| `dji_20260306_0016.json` | DJI 드론 정사 철도·산업 장면 (Grok+Gemini 병합) | +| `dji_20260306_0044.json` | **0016 확장** (`extends`) + scene_delta (철로+도로+열차) | + +## tier + +| tier | 내용 | +|------|------| +| `compact` | 1차 실험용 19개 (기본) | +| `gap` | **누락 보강 12개** (Gemini 재검토: road, dense forest, covered truck 등) | +| `gap2` | 합본 빈 영역 보강 (다수 프롬프트) | +| `gap2_core` | gap2 중 실측 성공 프롬프트만 (재실행 권장) | +| `improved` | compact + gap 병합 (~31개) | +| `A_high` | 1차 고확률 | +| `B_facility` | 시설·구조 | +| `C_detail` | 야적·지면·지붕색 | +| `D_rail_domain` | 철도 특화 (편차 큼) | +| `E_missed` | gap 확장판 (동의어 포함) | +| `all` | A→E 전부 (중복 제거) | + +## 기대 난이도 (정사 드론 기준) + +| 프롬프트 | 기대 | 튜닝 팁 | +|----------|------|---------| +| `building`, `solar panel`, `tree`, `car`, `truck` | 높음 | 기본 객체. 낮은 confidence에서도 비교적 잘 잡힘 | +| `railway track`, `fence`, `parking lot`, `blue tarp` | 중~높음 | 선형·구획. context에 따라 변동 | +| `utility pole`, `container`, `material pile` | 중 | 작거나 밀집. `--min-size`로 노이즈 제거 | +| `rail`, `ballast`, `railroad sleeper`, `catenary` | 낮~중 | 도메인 용어·가는 객체. confidence 튜닝 필수 | + +권장: **confidence 0.25~0.35** 로 시작. + +## 단일 프롬프트 (`text_segment.py`) + +```powershell +cd D:\MYCLAUDE_PROJECT\samgeo3-lab +.\.venv\Scripts\Activate.ps1 + +$img = "data\DJI_20260306100802_0016.JPG" +# 또는: $img = "D:\MYCLAUDE_PROJECT\segment-geospatial\sample\DJI_20260306100802_0016.JPG" +$out = "output\dji_0016_analysis" + +python scripts\text_segment.py ` + --image $img ` + --prompt "solar panel" ` + --confidence 0.3 ` + --output-dir $out + +python scripts\text_segment.py ` + --image $img ` + --prompt "building" ` + --confidence 0.3 ` + --output-dir $out + +python scripts\text_segment.py ` + --image $img ` + --prompt "railway track" ` + --confidence 0.3 ` + --output-dir $out +``` + +결과 예: `output\dji_0016_analysis\DJI_..._solar_panel_mask.png` + +## 멀티 프롬프트 일괄 (`multi_prompt_segment.py`) — 이미 포함됨 + +모델·이미지를 **1회 로드** 후 프롬프트 루프. 결과는 +`output\_\{stem}_{prompt}_mask.png` 등 + `summary.json`. + +```powershell +cd D:\MYCLAUDE_PROJECT\samgeo3-lab +.\.venv\Scripts\Activate.ps1 + +# 프롬프트 목록만 +python scripts\multi_prompt_segment.py --list-only --tier compact + +# compact 19개 일괄 (권장 1차) +python scripts\multi_prompt_segment.py ` + --image "data\DJI_20260306100802_0016.JPG" ` + --tier compact ` + --confidence 0.3 ` + --output-dir "output\dji_0016_compact" + +# 누락 보강 (Gemini gap) — confidence 약간 낮게, 같은 폴더에 추가하면 merge에 포함 +python scripts\multi_prompt_segment.py ` + --image "data\DJI_20260306100802_0016.JPG" ` + --tier gap ` + --confidence 0.25 ` + --output-dir "output\dji_0016_compact" + +# 1차+보강 한 번에 +python scripts\multi_prompt_segment.py --tier improved --confidence 0.28 + +# 1차 고확률만 +python scripts\multi_prompt_segment.py --tier A_high --confidence 0.3 + +# 전체 tier +python scripts\multi_prompt_segment.py --tier all --confidence 0.25 + +# 직접 지정 +python scripts\multi_prompt_segment.py ` + --prompts "solar panel,building,railway track,tree,car,truck" ` + --confidence 0.3 ` + --output-dir "output\dji_0016_analysis" +``` + +## DJI_0044 = 이미지 1번(0016) 확장 (별도 체계 아님) + +```text +0016 base (compact/gap/gap2) + scene_delta(열차·도로·야적) → 0044 +``` + +```powershell +# 상속 확인 +python scripts\multi_prompt_segment.py --prompts-json prompts\dji_20260306_0044.json --list-only --tier compact +python scripts\multi_prompt_segment.py --prompts-json prompts\dji_20260306_0044.json --list-only --tier delta + +# 0016과 동일한 파이프라인 +python scripts\multi_prompt_segment.py ` + --prompts-json prompts\dji_20260306_0044.json ` + --image data\DJI_20260306100928_0044.JPG ` + --tier compact --confidence 0.3 ` + --output-dir output\dji_0044_compact + +python scripts\multi_prompt_segment.py ` + --prompts-json prompts\dji_20260306_0044.json ` + --tier gap --confidence 0.25 ` + --output-dir output\dji_0044_compact + +python scripts\merge_multi_results.py ` + --result-dir output\dji_0044_compact ` + --image data\DJI_20260306100928_0044.JPG +``` + +HTML: `docs/dji_0044_prompts.html` + +## 예시: DJI_0100 전체 파이프라인 (compact → gap → gap2_core → merge) + +0044 프롬프트 JSON(0016 상속 + 도로/열차 delta)을 **0100 이미지**에 적용하는 +권장 end-to-end 예입니다. gap / gap2_core 단계는 선택입니다. + +```powershell +cd D:\MYCLAUDE_PROJECT\samgeo3-lab +.\.venv\Scripts\Activate.ps1 + +$img = "data\DJI_20260306101434_0100.JPG" +$out = "output\dji_0100_compact" +$json = "prompts\dji_20260306_0044.json" + +# (선택) 프롬프트 목록 확인 +python scripts\multi_prompt_segment.py --prompts-json $json --list-only --tier compact + +# 1) 세그 — compact (0016 상속 + 도로/열차 delta) +python scripts\multi_prompt_segment.py ` + --prompts-json $json ` + --image $img ` + --tier compact ` + --confidence 0.3 ` + --output-dir $out + +# 2) (선택) gap 보강 +python scripts\multi_prompt_segment.py ` + --prompts-json $json ` + --image $img ` + --tier gap ` + --confidence 0.25 ` + --output-dir $out + +# 3) (선택) gap2_core 보강 +python scripts\multi_prompt_segment.py ` + --prompts-json $json ` + --image $img ` + --tier gap2_core ` + --confidence 0.22 ` + --output-dir $out + +# 4) 합치기 + interactive index.html +python scripts\merge_multi_results.py ` + --result-dir $out ` + --image $img ` + --max-side 2048 + +# 5) 결과 열기 +start "$out\merged\index.html" +``` + +권장 confidence: compact `0.3` → gap `0.25` → gap2_core `0.22`. +같은 `$out` 폴더에 단계별로 마스크를 쌓은 뒤 merge하면 한 HTML에서 전부 볼 수 있습니다. + +## 결과 합치기 (overlay + grid + HTML) + +`multi_prompt_segment` 실행 후: + +```powershell +python scripts\merge_multi_results.py ` + --result-dir "output\dji_0016_compact" ` + --image "data\DJI_20260306100802_0016.JPG" ` + --max-side 2560 +``` + +출력 (`output\dji_0016_compact\merged\`): + +| 파일 | 내용 | +|------|------| +| `combined_overlay.png` | 전 클래스 색상 오버레이 | +| `combined_with_legend.png` | 오버레이 + 범례 | +| `per_class_grid.png` | 클래스별 썸네일 격자 | +| `index.html` | 브라우저로 한눈에 보기 | +| `merge_summary.json` | 클래스·픽셀 통계 | diff --git a/prompts/dji_20260306_0016.json b/prompts/dji_20260306_0016.json new file mode 100644 index 0000000..2f77e2a --- /dev/null +++ b/prompts/dji_20260306_0016.json @@ -0,0 +1,385 @@ +{ + "id": "dji_20260306_0016", + "image": "data/DJI_20260306100802_0016.JPG", + "image_abs_hint": "D:/MYCLAUDE_PROJECT/segment-geospatial/sample/DJI_20260306100802_0016.JPG", + "sources": ["grok-vision", "gemini-vision", "gemini-missed-review"], + "scene_notes_ko": { + "center": "복선 철로, 침목, 자갈 도상(발라스트), 전차선 전주(폴)", + "above_track": "나대지/낙엽 경사면, 검은 덮개막, 파란 타프", + "below_track": "녹색 방음벽/펜스 라인", + "top_left": "낙엽 수목 지역", + "top_right": "대형 창고·차양, 주차장, 흰 승용차, 덮개 화물차, 녹색 지붕 소형 건물, 파란 컨테이너, 노란 안전구역, 가로등/전주", + "bottom": "태양광 패널 지붕, 흰/회색 지붕, 파란/녹색 지붕, 자재 야적(철재·파이프), 녹색 방수시트, 드럼통 더미", + "missed_review_ko": "범례에 가려진 우측 숲·철로 연속부; parking lot만 되고 road 누락; covered truck; street light; drainage; debris" + }, + "tiers": { + "A_high": { + "description": "잘 잡힐 가능성 높음 — 강력한 기본 객체", + "prompts": [ + "railway track", + "train track", + "railroad", + "building", + "roof", + "solar panel", + "tree", + "car", + "truck", + "fence" + ] + }, + "B_facility": { + "description": "구체적 시설·구조", + "prompts": [ + "warehouse", + "factory building", + "parking lot", + "container", + "utility pole", + "electric pole", + "catenary pole", + "sound barrier", + "retaining wall", + "drainage ditch", + "canopy", + "awning" + ] + }, + "C_detail": { + "description": "세부·야적·지면·지붕 색", + "prompts": [ + "solar farm", + "photovoltaic panel", + "cargo truck", + "covered truck", + "material pile", + "pipe stack", + "steel pile", + "steel pipe", + "bare ground", + "dirt road", + "paved road", + "shed", + "small building", + "blue tarp", + "black tarp", + "green tarp", + "construction tarp", + "green roof", + "blue roof", + "white roof", + "parking space" + ] + }, + "D_rail_domain": { + "description": "철도 특화 — 성공 편차 큼, confidence/min_size 튜닝 필요", + "prompts": [ + "rail", + "ballast", + "railroad sleeper", + "overhead line", + "railway catenary", + "railway embankment", + "noise barrier wall" + ] + }, + "E_missed": { + "description": "Gemini 재검토: 1차 compact에서 놓치거나 가려진 영역 보강", + "review_notes_ko": [ + "범례로 가려진 우측: dense forest / tree 연속부", + "철로·도상 연속: railway 보강은 D와 중복 가능, gravel path 추가", + "parking lot 연결 아스팔트 road 누락", + "covered truck / cargo vehicle 누락", + "street light / lamp post, drainage ditch", + "debris / rubbish / industrial waste" + ], + "prompts": [ + "road", + "asphalt pavement", + "asphalt road", + "paved road", + "gravel path", + "dense forest", + "forest", + "covered truck", + "cargo vehicle", + "blue covered truck", + "street light", + "lamp post", + "street lamp", + "light pole", + "drainage ditch", + "drainage channel", + "debris", + "rubbish", + "industrial waste", + "scrap metal", + "scrap pile" + ], + "recommended_confidence": 0.25 + } + }, + "compact": { + "description": "1차 실험용 압축 세트", + "prompts": [ + "railway track", + "railroad", + "fence", + "utility pole", + "building", + "warehouse", + "solar panel", + "green roof", + "blue roof", + "white roof", + "car", + "truck", + "blue tarp", + "black tarp", + "container", + "material pile", + "tree", + "bare ground", + "parking lot" + ] + }, + "gap": { + "description": "누락 보강 전용 — Gemini 추천 + 실측 성공 동의어", + "prompts": [ + "road", + "asphalt pavement", + "gravel path", + "dense forest", + "covered truck", + "cargo vehicle", + "street light", + "lamp post", + "light pole", + "drainage ditch", + "debris", + "rubbish", + "industrial waste", + "scrap pile" + ], + "recommended_confidence": 0.25, + "notes_ko": { + "worked": ["road", "asphalt pavement", "gravel path", "dense forest", "covered truck", "cargo vehicle", "debris", "rubbish", "light pole", "scrap pile"], + "hard_at_0.25": ["street light", "lamp post", "drainage ditch", "industrial waste"], + "tip": "가로등은 light pole@0.2, 배수로는 정사에서 거의 안 잡힘, rubbish/scrap pile은 노이즈 많을 수 있음" + } + }, + "improved": { + "description": "compact + gap 병합 (1차+누락 보강, 중복 제거)", + "prompts": [ + "railway track", + "railroad", + "fence", + "utility pole", + "building", + "warehouse", + "solar panel", + "green roof", + "blue roof", + "white roof", + "car", + "truck", + "blue tarp", + "black tarp", + "container", + "material pile", + "tree", + "bare ground", + "parking lot", + "road", + "asphalt pavement", + "gravel path", + "dense forest", + "covered truck", + "cargo vehicle", + "street light", + "lamp post", + "light pole", + "drainage ditch", + "debris", + "rubbish", + "industrial waste", + "scrap pile" + ], + "recommended_confidence": 0.28 + }, + "gap2": { + "description": "combined_with_legend 재검토: 아직 원본이 비어 보이는 영역 보강", + "source": "visual review of final merged overlay (Downloads/combined_with_legend.png)", + "gaps_ko": [ + "선로 상·하 갈색 성토/제방·마른 초지(scrub) — railway 노란 밴드 바깥", + "검은 타프 주변 갈색 흙더미·절토면", + "주차장·창고 사이 미포장 공터·콘크리트 슬라브", + "하측 야적장 녹생 방수시트/덮개(green tarp) 일부 미채움", + "소형 부스·shed·담장·차양", + "금속/골판 지붕 중 색 라벨에 안 걸린 부분", + "전차선·가느다란 전선/폴 잔여", + "건물 벽면·그림자 밴드", + "선로 침목·도상(ballast) 세부" + ], + "prompts": [ + "embankment", + "railway embankment", + "dirt embankment", + "soil pile", + "dirt pile", + "earth mound", + "dry grass", + "scrub", + "bushes", + "grass", + "vegetation", + "dormant vegetation", + "brown field", + "open ground", + "concrete", + "concrete pad", + "concrete pavement", + "industrial yard", + "paved yard", + "green tarp", + "green plastic sheet", + "plastic cover", + "shed", + "small shed", + "booth", + "kiosk", + "metal roof", + "corrugated roof", + "gray roof", + "wall", + "building wall", + "fence wall", + "barrier", + "retaining wall", + "canopy", + "awning", + "ballast", + "track bed", + "railway ballast", + "power line", + "overhead wire", + "shadow", + "sidewalk", + "walkway" + ], + "recommended_confidence": 0.22, + "worked_at_0.22": [ + "dirt embankment", + "dirt pile", + "dry grass", + "bushes", + "grass", + "vegetation", + "brown field", + "open ground", + "concrete", + "concrete pad", + "concrete pavement", + "industrial yard", + "paved yard", + "green tarp", + "green plastic sheet", + "plastic cover", + "shed", + "small shed", + "metal roof", + "corrugated roof", + "gray roof", + "wall", + "building wall", + "fence wall", + "barrier", + "canopy", + "awning", + "sidewalk" + ], + "failed_at_0.22": [ + "embankment", + "railway embankment", + "soil pile", + "earth mound", + "scrub", + "dormant vegetation", + "booth", + "kiosk", + "retaining wall", + "ballast", + "track bed", + "railway ballast", + "power line", + "overhead wire", + "shadow", + "walkway" + ] + }, + "gap2_core": { + "description": "gap2 중 실측 성공 프롬프트만 (재실행 권장 세트)", + "prompts": [ + "dirt embankment", + "dirt pile", + "dry grass", + "bushes", + "vegetation", + "brown field", + "open ground", + "concrete", + "industrial yard", + "green tarp", + "plastic cover", + "shed", + "metal roof", + "corrugated roof", + "gray roof", + "building wall", + "barrier", + "awning" + ], + "recommended_confidence": 0.22 + }, + "recommended": { + "model_id": "facebook/sam3.1", + "backend": "meta", + "confidence": 0.3, + "confidence_range": [0.25, 0.35], + "gap_confidence": 0.25, + "min_size": 0, + "note": "1차는 compact@0.3, 누락 보강은 gap@0.25 후 merge. 작은 폴/자재는 --min-size." + }, + "expected_difficulty": [ + { + "prompts": ["building", "solar panel", "tree", "car", "truck"], + "level": "high", + "level_ko": "높음", + "tip_ko": "기본 객체. 낮은 confidence에서도 비교적 잘 잡힘." + }, + { + "prompts": ["railway track", "fence", "parking lot", "blue tarp"], + "level": "medium_high", + "level_ko": "중~높음", + "tip_ko": "선형·구획 객체. 장면 context에 따라 성능 변동." + }, + { + "prompts": ["road", "asphalt pavement", "dense forest", "covered truck"], + "level": "medium", + "level_ko": "중 (gap 보강)", + "tip_ko": "1차에서 자주 누락. gap tier로 재실행. confidence 0.25 권장." + }, + { + "prompts": ["utility pole", "container", "material pile", "street light", "lamp post"], + "level": "medium", + "level_ko": "중", + "tip_ko": "작거나 밀집. --min-size로 노이즈 제거 권장." + }, + { + "prompts": ["rail", "ballast", "railroad sleeper", "railway catenary", "debris", "rubbish"], + "level": "low_medium", + "level_ko": "낮~중", + "tip_ko": "도메인·가는 객체·잔해. confidence 튜닝 필수." + } + ] +} diff --git a/prompts/dji_20260306_0044.json b/prompts/dji_20260306_0044.json new file mode 100644 index 0000000..6e1e561 --- /dev/null +++ b/prompts/dji_20260306_0044.json @@ -0,0 +1,157 @@ +{ + "id": "dji_20260306_0044", + "extends": "prompts/dji_20260306_0016.json", + "extends_note_ko": "1번 이미지(DJI_0016) 프롬프트·tier·gap/gap2 워크플로를 그대로 상속. 이 파일은 장면 전용 객체만 추가 확장. 별도 체계 아님.", + "image": "data/DJI_20260306100928_0044.JPG", + "image_abs_hint": "D:/MYCLAUDE_PROJECT/segment-geospatial/sample/DJI_20260306100928_0044.JPG", + "sources": ["extends:dji_20260306_0016", "gemini-vision", "grok-vision"], + "scene_notes_ko": { + "extends": "0016 = 철로+산업단지 베이스. 0044 = 동일 파이프라인 + 도로·열차·야적 강화", + "top": "다차로 도로·교차로·횡단보도·주행 차량·가로수·밀집 건물·태양광", + "middle": "야적장·파이프·백색 자루·목재·철재·파란 타프·철조망 펜스", + "bottom": "복선 철로·열차(객차/화차)·전차선 폴·나대지" + }, + "scene_delta": { + "description": "0016 베이스에 없는(또는 이 장면에서 핵심인) 추가 객체 — Gemini+Grok", + "prompts": [ + "train", + "locomotive", + "passenger train", + "freight car", + "container car", + "highway", + "pedestrian crossing", + "crosswalk", + "road marking", + "chain-link fence", + "pipe stack", + "pipe pile", + "pipes", + "industrial pipes", + "bag pile", + "bulk bag", + "white sack pile", + "wooden pile", + "lumber pile", + "wood stack", + "tarp", + "gravel", + "gravel ground", + "paved ground", + "dirt ground", + "shrub", + "sleeper", + "trackbed", + "catenary", + "boxcar", + "construction material", + "industrial material", + "debris pile", + "red roof" + ], + "recommended_confidence": 0.28 + }, + "tiers": { + "A_high": { + "description": "0016 A_high + 도로/열차 핵심 추가", + "prompts": [ + "road", + "highway", + "train", + "freight car", + "container car", + "car", + "truck" + ] + }, + "B_facility": { + "description": "0016 B + 횡단보도·도로시설", + "prompts": [ + "pedestrian crossing", + "crosswalk", + "road marking", + "chain-link fence", + "street light", + "light pole" + ] + }, + "C_detail": { + "description": "0016 C + 야적 자재 세분", + "prompts": [ + "pipe pile", + "pipes", + "bag pile", + "wooden pile", + "wood stack", + "tarp", + "gravel", + "dirt ground" + ] + }, + "D_rail_domain": { + "description": "0016 D + 열차 관련", + "prompts": [ + "train", + "freight car", + "container car", + "sleeper", + "trackbed", + "catenary" + ] + } + }, + "compact": { + "description": "0016 compact 상속 + scene_delta(핵심만 필터하지 않고 전체 병합; 실행 시 길면 --tier delta)", + "include_scene_delta": true, + "prompts": [ + "train", + "freight car", + "container car", + "highway", + "pedestrian crossing", + "pipe pile", + "wooden pile", + "wood stack" + ], + "recommended_confidence": 0.3 + }, + "recommended": { + "model_id": "facebook/sam3.1", + "backend": "meta", + "confidence": 0.3, + "confidence_range": [0.25, 0.35], + "workflow_ko": [ + "1) 0016과 동일: compact → gap → gap2_core (상속된 tier)", + "2) 0044 전용만: --tier delta", + "3) 합본: base compact+delta = --tier compact", + "4) merge_multi_results.py" + ], + "note": "별도 체계 아님. 1번 이미지 확장." + }, + "expected_difficulty": [ + { + "prompts": ["building", "train", "car", "truck", "tree", "road"], + "level": "high", + "level_ko": "높음", + "tip_ko": "0016과 동일 orthophoto 기본 객체 + 열차" + }, + { + "prompts": ["railway track", "highway", "fence", "material pile", "pedestrian crossing"], + "level": "medium_high", + "level_ko": "중~높음", + "tip_ko": "선형·구획" + }, + { + "prompts": ["pipe pile", "bag pile", "wooden pile", "street light"], + "level": "medium", + "level_ko": "중", + "tip_ko": "밀집 자재 — min_size / conf 0.22~0.28" + }, + { + "prompts": ["rail", "ballast", "sleeper", "catenary", "locomotive"], + "level": "low_medium", + "level_ko": "낮~중", + "tip_ko": "도메인·세분 용어 — 0016 gap2와 동일 한계" + } + ] +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..20baae2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "samgeo3-lab" +version = "0.1.0" +description = "Independent SamGeo3 (SAM 3 / SAM 3.1) segmentation experiments" +readme = "README.md" +requires-python = ">=3.12" +license = { text = "MIT" } +dependencies = [ + "segment-geospatial[samgeo3]>=1.4.1", + "triton-windows>=3.3.0.post19; sys_platform == 'windows'", + "matplotlib>=3.8", +] + +[project.optional-dependencies] +notebook = ["ipykernel>=6.29", "jupyterlab>=4.0"] + +[tool.uv] +# Torch CUDA wheels are installed separately from the pytorch index. +# See README install section. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c576a2e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +# Pin loosely for experiments. Install torch CUDA wheels FIRST (see README). +# uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121 +segment-geospatial[samgeo3]>=1.4.1 +triton-windows>=3.3.0.post19; sys_platform == "windows" +matplotlib>=3.8 +ipykernel>=6.29 +jupyterlab>=4.0 diff --git a/scripts/check_install.py b/scripts/check_install.py new file mode 100644 index 0000000..041d16c --- /dev/null +++ b/scripts/check_install.py @@ -0,0 +1,174 @@ +"""Verify SamGeo3 / PyTorch / CUDA install for this isolated lab env. + +Usage (from project root, with venv active): + python scripts/check_install.py + python scripts/check_install.py --load-model +""" + +from __future__ import annotations + +import argparse +import os +import sys +import traceback + + +def _ok(msg: str) -> None: + print(f"[OK] {msg}") + + +def _warn(msg: str) -> None: + print(f"[WARN] {msg}") + + +def _fail(msg: str) -> None: + print(f"[FAIL] {msg}") + + +def check_torch() -> bool: + try: + import torch + except ImportError as e: + _fail(f"torch import failed: {e}") + return False + + _ok(f"torch {torch.__version__}") + cuda = torch.cuda.is_available() + if cuda: + name = torch.cuda.get_device_name(0) + cap = torch.cuda.get_device_capability(0) + _ok(f"CUDA available | GPU={name} | capability={cap}") + _ok(f"cuda runtime reported by torch: {torch.version.cuda}") + else: + _warn("CUDA not available — SAM3 meta backend needs NVIDIA GPU + CUDA torch") + return True + + +def check_samgeo3_imports() -> bool: + try: + import samgeo + from samgeo import SamGeo3 + except ImportError as e: + _fail(f"samgeo / SamGeo3 import failed: {e}") + return False + + _ok(f"samgeo {getattr(samgeo, '__version__', 'unknown')}") + _ok("SamGeo3 class importable") + + try: + import sam3 # noqa: F401 + + _ok(f"sam3 package present ({getattr(sam3, '__version__', 'no __version__')})") + except ImportError as e: + _fail(f"sam3 package missing (required for backend='meta'): {e}") + return False + + return True + + +def check_hf_access() -> None: + token_env = bool(os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")) + token_file = os.path.expanduser("~/.cache/huggingface/token") + has_file = os.path.isfile(token_file) + if token_env: + _ok("HF token found in environment") + elif has_file: + _ok(f"HF token file present: {token_file}") + else: + _warn( + "No HF token found. SAM3/3.1 gated models need: " + "hf auth login (after HF access approval)" + ) + + ckpt = os.environ.get("SAM3_CHECKPOINT_PATH") + if ckpt: + exists = os.path.isfile(ckpt) + ( _ok if exists else _fail)(f"SAM3_CHECKPOINT_PATH={ckpt} exists={exists}") + else: + _warn("SAM3_CHECKPOINT_PATH not set (optional; HF download/cache used otherwise)") + + +def try_load_model(model_id: str, device: str | None) -> bool: + try: + from samgeo import SamGeo3 + import torch + except ImportError as e: + _fail(f"imports for model load failed: {e}") + return False + + if device is None: + device = "cuda" if torch.cuda.is_available() else "cpu" + + if device == "cuda" and not torch.cuda.is_available(): + _fail("requested CUDA but torch.cuda.is_available() is False") + return False + + print(f"\nLoading SamGeo3(backend='meta', model_id='{model_id}', device='{device}') ...") + try: + kwargs = { + "backend": "meta", + "model_id": model_id, + "device": device, + "confidence_threshold": 0.5, + "enable_segmentation": True, + "enable_inst_interactivity": False, + } + ckpt = os.environ.get("SAM3_CHECKPOINT_PATH") + if ckpt and os.path.isfile(ckpt): + kwargs["checkpoint_path"] = ckpt + kwargs["load_from_HF"] = False + _ok(f"using local checkpoint: {ckpt}") + + sam = SamGeo3(**kwargs) + _ok(f"model loaded | backend={sam.backend} | device={getattr(sam, 'device', device)}") + del sam + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return True + except Exception as e: + _fail(f"model load failed: {e}") + traceback.print_exc() + return False + + +def main() -> int: + parser = argparse.ArgumentParser(description="SamGeo3 install verification") + parser.add_argument( + "--load-model", + action="store_true", + help="Also instantiate SamGeo3 (downloads weights if needed)", + ) + parser.add_argument( + "--model-id", + default="facebook/sam3.1", + help="HF model id for --load-model (default: facebook/sam3.1)", + ) + parser.add_argument( + "--device", + default=None, + help="cuda | cpu (default: auto)", + ) + args = parser.parse_args() + + print("=== SamGeo3 lab: install check ===\n") + print(f"python: {sys.version}") + print(f"exe: {sys.executable}\n") + + ok = True + ok = check_torch() and ok + ok = check_samgeo3_imports() and ok + check_hf_access() + + if args.load_model: + ok = try_load_model(args.model_id, args.device) and ok + + print() + if ok: + _ok("all required checks passed") + return 0 + _fail("one or more required checks failed") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/merge_multi_results.py b/scripts/merge_multi_results.py new file mode 100644 index 0000000..80830aa --- /dev/null +++ b/scripts/merge_multi_results.py @@ -0,0 +1,1100 @@ +"""Merge multi-prompt SamGeo3 results into one overview (overlay + grid + HTML). + +Works on the output directory from multi_prompt_segment.py +(e.g. output/dji_0016_compact with *_mask.png files). + +Usage (from project root, venv active): + python scripts/merge_multi_results.py --result-dir output/dji_0016_compact + python scripts/merge_multi_results.py --result-dir output/dji_0016_compact --image data/DJI_....JPG + python scripts/merge_multi_results.py --result-dir output/dji_0016_compact --max-side 2048 +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image, ImageDraw, ImageFont + +ROOT = Path(__file__).resolve().parents[1] + +# Distinct RGB colors for class overlays (cycled if more prompts) +PALETTE = [ + (230, 25, 75), # red + (60, 180, 75), # green + (0, 130, 200), # blue + (245, 130, 48), # orange + (145, 30, 180), # purple + (70, 240, 240), # cyan + (240, 50, 230), # magenta + (210, 245, 60), # lime + (250, 190, 212), # pink + (0, 128, 128), # teal + (220, 190, 255), # lavender + (170, 110, 40), # brown + (255, 250, 200), # beige + (128, 0, 0), # maroon + (170, 255, 195), # mint + (128, 128, 0), # olive + (255, 215, 180), # apricot + (0, 0, 128), # navy + (128, 128, 128), # gray + (255, 225, 25), # yellow +] + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Merge multi-prompt segmentation results") + p.add_argument( + "--result-dir", + type=Path, + required=True, + help="Directory with *_mask.png (and optional summary.json)", + ) + p.add_argument( + "--image", + type=Path, + default=None, + help="Original image (else from summary.json or data/ fallback)", + ) + p.add_argument( + "--output-dir", + type=Path, + default=None, + help="Where to write merged outputs (default: /merged)", + ) + p.add_argument( + "--max-side", + type=int, + default=2560, + help="Max width/height of overview images (keeps memory/file size down)", + ) + p.add_argument( + "--alpha", + type=float, + default=0.45, + help="Mask overlay opacity 0~1", + ) + p.add_argument( + "--grid-cols", + type=int, + default=4, + help="Columns in thumbnail grid", + ) + p.add_argument( + "--thumb-size", + type=int, + default=480, + help="Thumbnail max side for grid cells", + ) + p.add_argument( + "--no-grid", + action="store_true", + help="Skip per-class grid image", + ) + p.add_argument( + "--no-html", + action="store_true", + help="Skip HTML report", + ) + return p.parse_args() + + +def load_summary(result_dir: Path) -> dict[str, Any] | None: + path = result_dir / "summary.json" + if not path.is_file(): + return None + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _common_prefix(strings: list[str]) -> str: + if not strings: + return "" + prefix = strings[0] + for s in strings[1:]: + while not s.startswith(prefix) and prefix: + prefix = prefix[:-1] + if not prefix: + break + return prefix + + +def discover_masks(result_dir: Path) -> list[dict[str, Any]]: + """Return list of {prompt, mask_path, ann_path, n_from_scores} from files.""" + masks = sorted(result_dir.glob("*_mask.png")) + bases = [mp.name[: -len("_mask.png")] for mp in masks] + # e.g. common "DJI_..._0016_" then remainder is prompt tag (may contain _) + cpref = _common_prefix(bases) + # prefer cut at last underscore of common prefix so prompt is clean + if cpref and not cpref.endswith("_"): + # trim to last underscore so we don't eat prompt chars + li = cpref.rfind("_") + cpref = cpref[: li + 1] if li >= 0 else "" + + items: list[dict[str, Any]] = [] + for mp, base in zip(masks, bases): + tag = base[len(cpref) :] if cpref and base.startswith(cpref) else base + if not tag: + # fallback: last underscore segment only + tag = base.rsplit("_", 1)[-1] + prompt = tag.replace("_", " ") + ann = result_dir / f"{base}_ann.png" + scores = result_dir / f"{base}_scores.npy" + n_obj = None + if scores.is_file(): + try: + n_obj = int(np.load(scores).shape[0]) + except Exception: + n_obj = None + items.append( + { + "prompt": prompt, + "tag": tag, + "base": base, + "mask_path": mp, + "ann_path": ann if ann.is_file() else None, + "scores_path": scores if scores.is_file() else None, + "n_objects": n_obj, + } + ) + return items + + +def items_from_summary(summary: dict[str, Any], result_dir: Path) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + for r in summary.get("results", []): + n = int(r.get("n_objects") or 0) + if n <= 0 and not r.get("mask"): + continue + mask_s = r.get("mask") + mask_path = Path(mask_s) if mask_s else None + if mask_path is None or not mask_path.is_file(): + # try local name + prompt = r.get("prompt", "") + tag = "".join(c if c.isalnum() or c in "-_" else "_" for c in prompt) + cands = list(result_dir.glob(f"*_{tag}_mask.png")) + mask_path = cands[0] if cands else None + if mask_path is None or not mask_path.is_file(): + continue + ann_s = r.get("ann") + ann_path = Path(ann_s) if ann_s and Path(ann_s).is_file() else None + items.append( + { + "prompt": r.get("prompt", mask_path.stem), + "tag": mask_path.stem.replace("_mask", "").split("_")[-1], + "base": mask_path.name[: -len("_mask.png")], + "mask_path": mask_path, + "ann_path": ann_path, + "scores_path": Path(r["scores_npy"]) + if r.get("scores_npy") and Path(r["scores_npy"]).is_file() + else None, + "n_objects": n, + "scores_min": r.get("scores_min"), + "scores_max": r.get("scores_max"), + } + ) + return items + + +def resolve_base_image( + args_image: Path | None, + summary: dict[str, Any] | None, + result_dir: Path, +) -> Path: + cands: list[Path] = [] + if args_image: + cands.append(args_image) + if summary and summary.get("image"): + cands.append(Path(summary["image"])) + # common lab defaults + cands.append(ROOT / "data" / "DJI_20260306100802_0016.JPG") + cands.append( + Path(r"D:\MYCLAUDE_PROJECT\segment-geospatial\sample\DJI_20260306100802_0016.JPG") + ) + for c in cands: + if c and Path(c).is_file(): + return Path(c).resolve() + raise FileNotFoundError( + "Base image not found. Pass --image. Tried:\n " + + "\n ".join(str(c) for c in cands) + ) + + +def resize_max(im: Image.Image, max_side: int) -> Image.Image: + w, h = im.size + m = max(w, h) + if m <= max_side: + return im + scale = max_side / m + nw, nh = int(w * scale), int(h * scale) + return im.resize((nw, nh), Image.Resampling.BILINEAR) + + +def load_mask_bool(path: Path, size: tuple[int, int]) -> np.ndarray: + m = Image.open(path) + if m.mode not in ("L", "I", "I;16", "P"): + m = m.convert("L") + if m.size != size: + m = m.resize(size, Image.Resampling.NEAREST) + arr = np.array(m) + return arr > 0 + + +def try_font(size: int) -> ImageFont.ImageFont: + for name in ( + "C:/Windows/Fonts/malgun.ttf", + "C:/Windows/Fonts/segoeui.ttf", + "C:/Windows/Fonts/arial.ttf", + ): + if Path(name).is_file(): + try: + return ImageFont.truetype(name, size) + except Exception: + pass + return ImageFont.load_default() + + +def build_overlay( + base_rgb: Image.Image, + items: list[dict[str, Any]], + alpha: float, +) -> tuple[Image.Image, list[dict[str, Any]]]: + base = np.asarray(base_rgb.convert("RGB"), dtype=np.float32) + h, w = base.shape[:2] + out = base.copy() + legend: list[dict[str, Any]] = [] + + for i, it in enumerate(items): + color = PALETTE[i % len(PALETTE)] + try: + mask = load_mask_bool(it["mask_path"], (w, h)) + except Exception as e: + print(f" [WARN] skip mask {it['mask_path'].name}: {e}") + continue + pix = int(mask.sum()) + if pix == 0: + continue + c = np.array(color, dtype=np.float32) + out[mask] = out[mask] * (1.0 - alpha) + c * alpha + legend.append( + { + "prompt": it["prompt"], + "color": color, + "n_objects": it.get("n_objects"), + "pixels": pix, + "scores_min": it.get("scores_min"), + "scores_max": it.get("scores_max"), + } + ) + + blended = Image.fromarray(np.clip(out, 0, 255).astype(np.uint8)) + return blended, legend + + +def draw_legend_panel( + legend: list[dict[str, Any]], + width: int = 420, + row_h: int = 36, +) -> Image.Image: + font = try_font(16) + title_font = try_font(20) + n = max(len(legend), 1) + height = 56 + n * row_h + 16 + panel = Image.new("RGB", (width, height), (24, 28, 36)) + draw = ImageDraw.Draw(panel) + draw.text((16, 14), "Merged classes (prompt)", fill=(240, 244, 248), font=title_font) + y = 52 + for ent in legend: + r, g, b = ent["color"] + draw.rectangle([16, y + 4, 40, y + 28], fill=(r, g, b)) + n_obj = ent.get("n_objects") + n_txt = f"n={n_obj}" if n_obj is not None else f"px={ent['pixels']}" + smin, smax = ent.get("scores_min"), ent.get("scores_max") + if smin is not None and smax is not None: + score_txt = f" conf {smin:.2f}~{smax:.2f}" + else: + score_txt = "" + label = f"{ent['prompt']} ({n_txt}{score_txt})" + draw.text((52, y + 6), label, fill=(220, 226, 234), font=font) + y += row_h + return panel + + +def compose_with_legend(overlay: Image.Image, legend_panel: Image.Image) -> Image.Image: + gap = 16 + w = overlay.width + gap + legend_panel.width + h = max(overlay.height, legend_panel.height) + canvas = Image.new("RGB", (w, h), (18, 20, 26)) + canvas.paste(overlay, (0, 0)) + canvas.paste(legend_panel, (overlay.width + gap, 0)) + return canvas + + +def build_grid( + base: Image.Image, + items: list[dict[str, Any]], + colors: list[tuple[int, int, int]], + cols: int, + thumb: int, + alpha: float, +) -> Image.Image: + font = try_font(18) + cells: list[Image.Image] = [] + for i, it in enumerate(items): + color = colors[i % len(colors)] + # small overlay per class + b = resize_max(base.copy(), thumb) + arr = np.asarray(b.convert("RGB"), dtype=np.float32) + h, w = arr.shape[:2] + try: + mask = load_mask_bool(it["mask_path"], (w, h)) + except Exception: + mask = np.zeros((h, w), dtype=bool) + c = np.array(color, dtype=np.float32) + if mask.any(): + arr[mask] = arr[mask] * (1.0 - alpha) + c * alpha + cell = Image.fromarray(np.clip(arr, 0, 255).astype(np.uint8)) + # title bar + bar_h = 32 + framed = Image.new("RGB", (cell.width, cell.height + bar_h), (30, 34, 42)) + framed.paste(cell, (0, bar_h)) + draw = ImageDraw.Draw(framed) + draw.rectangle([0, 0, framed.width, bar_h], fill=(30, 34, 42)) + draw.rectangle([8, 8, 24, 24], fill=color) + n = it.get("n_objects") + title = f"{it['prompt']}" + (f" (n={n})" if n is not None else "") + draw.text((32, 6), title[:40], fill=(235, 240, 245), font=font) + cells.append(framed) + + if not cells: + return Image.new("RGB", (thumb, thumb), (40, 40, 40)) + + cols = max(1, cols) + rows = (len(cells) + cols - 1) // cols + cw = max(c.width for c in cells) + ch = max(c.height for c in cells) + pad = 8 + grid_w = cols * cw + (cols + 1) * pad + grid_h = rows * ch + (rows + 1) * pad + grid = Image.new("RGB", (grid_w, grid_h), (18, 20, 26)) + for i, cell in enumerate(cells): + r, c = divmod(i, cols) + x = pad + c * (cw + pad) + y = pad + r * (ch + pad) + grid.paste(cell, (x, y)) + return grid + + +def enrich_item_scores(items: list[dict[str, Any]]) -> None: + """Fill scores_min/max/mean/n_objects from *_scores.npy when present.""" + for it in items: + sp = it.get("scores_path") + if sp is None or not Path(sp).is_file(): + continue + try: + arr = np.load(sp).astype(np.float64).ravel() + if arr.size == 0: + continue + it["n_objects"] = int(arr.size) + it["scores_min"] = float(arr.min()) + it["scores_max"] = float(arr.max()) + it["scores_mean"] = float(arr.mean()) + except Exception: + continue + + +def mask_to_rle(mask: np.ndarray) -> list[int]: + """Run-length encode binary mask (row-major). [start, length, start, length, ...]. + + Used so the browser can hit-test without canvas getImageData (file:// safe). + """ + flat = np.ascontiguousarray(mask.astype(bool).ravel()) + n = int(flat.size) + if n == 0: + return [] + # transitions where value changes + # find runs of True + rle: list[int] = [] + i = 0 + while i < n: + if flat[i]: + start = i + while i < n and flat[i]: + i += 1 + rle.append(int(start)) + rle.append(int(i - start)) + else: + i += 1 + return rle + + +def export_hitmasks( + out_dir: Path, + ordered_items: list[dict[str, Any]], + legend: list[dict[str, Any]], + size: tuple[int, int], +) -> list[dict[str, Any]]: + """Build interactive class meta with RLE hit data (file:// / no CORS needed).""" + w, h = size + classes_meta: list[dict[str, Any]] = [] + for i, ent in enumerate(legend): + it = None + for cand in ordered_items: + if cand["prompt"] == ent["prompt"] or cand["prompt"].replace( + "_", " " + ) == ent["prompt"]: + it = cand + break + if it is None and i < len(ordered_items): + it = ordered_items[i] + if it is None: + continue + + mask = load_mask_bool(it["mask_path"], (w, h)) + rle = mask_to_rle(mask) + + r, g, b = ent["color"] + smin = ent.get("scores_min") + if smin is None: + smin = it.get("scores_min") + smax = ent.get("scores_max") + if smax is None: + smax = it.get("scores_max") + smean = it.get("scores_mean") + if smean is None and smin is not None and smax is not None: + smean = (float(smin) + float(smax)) / 2.0 + + classes_meta.append( + { + "id": i, + "prompt": ent["prompt"], + "color": [int(r), int(g), int(b)], + "n_objects": ent.get("n_objects") + if ent.get("n_objects") is not None + else it.get("n_objects"), + "pixels": int(ent.get("pixels") or int(mask.sum())), + "scores_min": smin, + "scores_max": smax, + "scores_mean": smean, + "rle": rle, + } + ) + + meta = { + "width": w, + "height": h, + "overlay": "combined_overlay.png", + "hit_mode": "rle", + "classes": classes_meta, + } + meta_path = out_dir / "interactive_meta.json" + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f, ensure_ascii=False, indent=2) + return classes_meta + + +def write_html( + out_path: Path, + overlay_name: str, + grid_name: str | None, + legend: list[dict[str, Any]], + meta: dict[str, Any], + interactive_meta: dict[str, Any] | None = None, +) -> None: + """Write interactive index.html (click image → class list + scores).""" + # Embed meta so file:// works without fetch CORS issues + imeta = interactive_meta + if imeta is None: + imeta_path = out_path.parent / "interactive_meta.json" + if imeta_path.is_file(): + with open(imeta_path, encoding="utf-8") as f: + imeta = json.load(f) + else: + imeta = { + "width": 0, + "height": 0, + "overlay": overlay_name, + "classes": [], + } + imeta_js = json.dumps(imeta, ensure_ascii=False) + + # Static fallback rows still useful if JS fails + rows = [] + for ent in legend: + r, g, b = ent["color"] + n = ent.get("n_objects") + smin, smax = ent.get("scores_min"), ent.get("scores_max") + score_cell = ( + f"{smin:.3f}~{smax:.3f}" if smin is not None and smax is not None else "-" + ) + rows.append( + f"" + f"" + f"{ent['prompt']}" + f"{n if n is not None else '-'}" + f"{score_cell}" + f"{ent['pixels']:,}" + ) + grid_block = ( + f'
Per-class grid' + f'grid
' + if grid_name + else "" + ) + n_cls = len(legend) + html = f""" + + + + +Merged multi-prompt — interactive + + + +
+

Merged multi-prompt — click to inspect

+

이미지를 클릭하면 해당 픽셀에 겹친 클래스(프롬프트)와 confidence 점수가 오른쪽에 표시됩니다. 겹침 시 점수 높은 순으로 정렬됩니다.

+
+
+
+
+ +
+

클릭: 픽셀 조회 · Shift+클릭: 선택 유지(누적) · Esc: 초기화 · 클래스 {n_cls}개

+

loading…

+ {grid_block} +
+ +
+ + + +""" + out_path.write_text(html, encoding="utf-8") + + +def main() -> int: + args = parse_args() + result_dir = args.result_dir.resolve() + if not result_dir.is_dir(): + print(f"ERROR: result-dir not found: {result_dir}", file=sys.stderr) + return 2 + + summary = load_summary(result_dir) + # Prefer all masks on disk (summary.json is only the *last* multi run) + disk_items = discover_masks(result_dir) + if summary: + meta_by_prompt = { + r.get("prompt"): r for r in summary.get("results", []) if r.get("prompt") + } + for it in disk_items: + # exact or underscore-normalized match + meta = meta_by_prompt.get(it["prompt"]) + if meta is None: + meta = meta_by_prompt.get(it["prompt"].replace(" ", "_")) + if meta is None: + for k, v in meta_by_prompt.items(): + if k.replace(" ", "_") == it["tag"] or k == it["tag"].replace( + "_", " " + ): + meta = v + break + if meta: + if it.get("n_objects") is None and meta.get("n_objects") is not None: + it["n_objects"] = meta.get("n_objects") + it["scores_min"] = meta.get("scores_min") + it["scores_max"] = meta.get("scores_max") + items = disk_items + # also include any summary-only masks not found by discover + if not items: + items = items_from_summary(summary, result_dir) + else: + items = disk_items + + # only keep masks that have content + filtered: list[dict[str, Any]] = [] + for it in items: + try: + m = Image.open(it["mask_path"]) + arr = np.array(m.convert("L") if m.mode != "L" else m) + if (arr > 0).any(): + filtered.append(it) + except Exception: + continue + items = filtered + enrich_item_scores(items) + + if not items: + print(f"ERROR: no non-empty mask files in {result_dir}", file=sys.stderr) + return 1 + + try: + image_path = resolve_base_image(args.image, summary, result_dir) + except FileNotFoundError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 2 + + out_dir = (args.output_dir or (result_dir / "merged")).resolve() + out_dir.mkdir(parents=True, exist_ok=True) + + print("=== merge multi-prompt results ===") + print(f"result_dir: {result_dir}") + print(f"image: {image_path}") + print(f"classes: {len(items)}") + print(f"output: {out_dir}") + + base_full = Image.open(image_path).convert("RGB") + base = resize_max(base_full, args.max_side) + print(f"base size: {base_full.size} -> overview {base.size}") + + # resize masks via load_mask_bool to base.size + overlay, legend = build_overlay(base, items, alpha=args.alpha) + if not legend: + print("ERROR: all masks empty after resize", file=sys.stderr) + return 1 + + # propagate scores from items into legend + by_prompt = {it["prompt"]: it for it in items} + for ent in legend: + it = by_prompt.get(ent["prompt"]) + if not it: + continue + for k in ("scores_min", "scores_max", "scores_mean", "n_objects"): + if ent.get(k) is None and it.get(k) is not None: + ent[k] = it[k] + + legend_panel = draw_legend_panel(legend) + combined = compose_with_legend(overlay, legend_panel) + + overlay_path = out_dir / "combined_overlay.png" + with_legend_path = out_dir / "combined_with_legend.png" + overlay.save(overlay_path, optimize=True) + combined.save(with_legend_path, optimize=True) + print(f"saved: {overlay_path}") + print(f"saved: {with_legend_path}") + + # ordered items matching legend (for hitmasks + grid) + ordered = [] + for ent in legend: + for it in items: + if it["prompt"] == ent["prompt"] or it["prompt"].replace( + "_", " " + ) == ent["prompt"]: + ordered.append(it) + break + if not ordered: + ordered = items + + grid_name = None + if not args.no_grid: + colors = [PALETTE[i % len(PALETTE)] for i in range(len(ordered))] + grid = build_grid( + base_full, + ordered, + colors, + cols=args.grid_cols, + thumb=args.thumb_size, + alpha=args.alpha, + ) + grid_path = out_dir / "per_class_grid.png" + grid.save(grid_path, optimize=True) + grid_name = grid_path.name + print(f"saved: {grid_path}") + + # RLE hit data for interactive click query (file:// safe, no getImageData) + print("exporting RLE hit data for interactive HTML...") + classes_meta = export_hitmasks(out_dir, ordered, legend, base.size) + print(f"saved: {out_dir / 'interactive_meta.json'} ({len(classes_meta)} classes, RLE)") + + merge_summary = { + "result_dir": str(result_dir), + "image": str(image_path), + "n_classes": len(legend), + "max_side": args.max_side, + "alpha": args.alpha, + "overview_size": list(base.size), + "classes": legend, + "outputs": { + "combined_overlay": str(overlay_path), + "combined_with_legend": str(with_legend_path), + "per_class_grid": str(out_dir / "per_class_grid.png") + if grid_name + else None, + "interactive_meta": str(out_dir / "interactive_meta.json"), + "index_html": str(out_dir / "index.html"), + }, + } + sum_path = out_dir / "merge_summary.json" + with open(sum_path, "w", encoding="utf-8") as f: + json.dump(merge_summary, f, ensure_ascii=False, indent=2) + print(f"saved: {sum_path}") + + if not args.no_html: + html_path = out_dir / "index.html" + interactive_meta = { + "width": base.size[0], + "height": base.size[1], + "overlay": overlay_path.name, + "classes": classes_meta, + } + # Use overlay without side legend so click coords map 1:1 to hitmasks + write_html( + html_path, + overlay_name=overlay_path.name, + grid_name=grid_name, + legend=legend, + meta=merge_summary, + interactive_meta=interactive_meta, + ) + print(f"saved: {html_path} (interactive click inspect)") + + print("\nClasses merged:") + for ent in legend: + n = ent.get("n_objects") + print(f" - {ent['prompt']}: n={n} px={ent['pixels']:,}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/multi_prompt_segment.py b/scripts/multi_prompt_segment.py new file mode 100644 index 0000000..85fc8e4 --- /dev/null +++ b/scripts/multi_prompt_segment.py @@ -0,0 +1,570 @@ +"""Run SamGeo3 text segmentation for multiple prompts (one model load). + +Prompt sets are Grok + Gemini vision merged lists (see prompts/*.json). + +Usage (from project root, venv active): + python scripts/multi_prompt_segment.py + python scripts/multi_prompt_segment.py --tier compact + python scripts/multi_prompt_segment.py --tier A_high + python scripts/multi_prompt_segment.py --tier all + python scripts/multi_prompt_segment.py --prompts "building,solar panel,tree" + python scripts/multi_prompt_segment.py --image data/DJI_20260306100802_0016.JPG --tier compact +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_PROMPTS_JSON = ROOT / "prompts" / "dji_20260306_0016.json" +DEFAULT_IMAGE = ROOT / "data" / "DJI_20260306100802_0016.JPG" +# fallback to sample folder outside lab if data copy missing +SAMPLE_FALLBACK = Path( + r"D:\MYCLAUDE_PROJECT\segment-geospatial\sample\DJI_20260306100802_0016.JPG" +) + + +def safe_name(prompt: str) -> str: + return "".join(c if c.isalnum() or c in "-_" else "_" for c in prompt.strip()) + + +def _dedupe(prompts: list[str]) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for p in prompts: + p = p.strip() + if p and p not in seen: + seen.add(p) + out.append(p) + return out + + +def _merge_prompt_lists(*lists: list[str]) -> list[str]: + merged: list[str] = [] + for lst in lists: + merged.extend(lst or []) + return _dedupe(merged) + + +def load_prompt_config(path: Path, _stack: list[Path] | None = None) -> dict[str, Any]: + """Load JSON; if 'extends' is set, inherit base (image-1) and apply scene delta. + + Extension model (image N expands image 1, not a separate system): + - base tiers / compact / gap / improved / gap2* are kept + - child may add scene_delta.prompts and optional tier/set additions + - child image path overrides base image + """ + path = path.resolve() + stack = _stack or [] + if path in stack: + raise ValueError(f"Circular extends: {path}") + stack = stack + [path] + + with open(path, encoding="utf-8") as f: + cfg = json.load(f) + + extends = cfg.get("extends") + if not extends: + return cfg + + base_path = Path(extends) + if not base_path.is_absolute(): + # relative to project root first, then to this file's dir + cand = (ROOT / extends).resolve() + if not cand.is_file(): + cand = (path.parent / extends).resolve() + base_path = cand + if not base_path.is_file(): + raise FileNotFoundError(f"extends not found: {extends} (from {path})") + + base = load_prompt_config(base_path, stack) + + # Start from base, then overlay child metadata / image + merged: dict[str, Any] = json.loads(json.dumps(base)) # deep copy + merged["id"] = cfg.get("id", base.get("id")) + merged["extends"] = str(extends) + merged["extends_resolved"] = str(base_path) + for key in ( + "image", + "image_abs_hint", + "sources", + "scene_notes_ko", + "recommended", + "expected_difficulty", + ): + if key in cfg: + merged[key] = cfg[key] + + # scene-only additions (this image expands base) + delta = list(cfg.get("scene_delta", {}).get("prompts", [])) + merged["scene_delta"] = cfg.get("scene_delta", {"prompts": delta}) + + # Merge tiers: base list + child additions (if child redefines tier fully, use add only + # unless replace_tiers=true) + child_tiers = cfg.get("tiers", {}) + if cfg.get("replace_tiers"): + merged["tiers"] = child_tiers + else: + base_tiers = dict(merged.get("tiers", {})) + for tname, tblock in child_tiers.items(): + add = list(tblock.get("prompts", [])) + if tname in base_tiers: + base_tiers[tname] = { + **base_tiers[tname], + "description": tblock.get( + "description", base_tiers[tname].get("description", "") + ), + "prompts": _merge_prompt_lists( + base_tiers[tname].get("prompts", []), add + ), + } + else: + base_tiers[tname] = tblock + # also append scene_delta into A_high by default for visibility + if delta and "A_high" in base_tiers: + base_tiers["A_high"] = { + **base_tiers["A_high"], + "prompts": _merge_prompt_lists( + base_tiers["A_high"].get("prompts", []), delta + ), + } + merged["tiers"] = base_tiers + + # Named sets: inherit base, extend with delta / child prompts + for set_name in ( + "compact", + "gap", + "improved", + "gap2", + "gap2_core", + "compact_plus", + ): + base_set = merged.get(set_name, {}) + base_prompts = list(base_set.get("prompts", [])) if isinstance(base_set, dict) else [] + child_set = cfg.get(set_name) + if child_set is None: + # default: base set + scene_delta for compact / improved + if set_name in ("compact", "improved", "compact_plus") and delta: + merged[set_name] = { + **(base_set if isinstance(base_set, dict) else {}), + "description": ( + f"extends base {set_name} + scene_delta " + f"({cfg.get('id', 'child')})" + ), + "prompts": _merge_prompt_lists(base_prompts, delta), + } + continue + if child_set.get("mode") == "replace": + merged[set_name] = child_set + else: + # default mode: append (base + child + optional delta) + extra = list(child_set.get("prompts", [])) + use_delta = child_set.get("include_scene_delta", True) + parts = [base_prompts, extra] + if use_delta: + parts.append(delta) + merged[set_name] = { + **(base_set if isinstance(base_set, dict) else {}), + **{k: v for k, v in child_set.items() if k != "prompts"}, + "description": child_set.get( + "description", + f"extends base {set_name} + additions", + ), + "prompts": _merge_prompt_lists(*parts), + } + + # If child has no compact at all, still attach delta + if "compact" not in cfg and delta: + bp = list(base.get("compact", {}).get("prompts", [])) + merged["compact"] = { + **base.get("compact", {}), + "description": "base compact + scene_delta", + "prompts": _merge_prompt_lists(bp, delta), + } + + merged["inheritance_ko"] = ( + "1번 이미지 프롬프트 체계를 상속하고, 이 장면 전용 객체만 추가 확장." + ) + return merged + + +def resolve_prompts( + cfg: dict[str, Any], + tier: str, + extra_prompts: list[str] | None, +) -> list[str]: + if extra_prompts: + return [p.strip() for p in extra_prompts if p.strip()] + + # scene_delta only (this image's new objects vs base) + if tier in ("delta", "scene_delta", "new"): + return list(cfg.get("scene_delta", {}).get("prompts", [])) + + # top-level named sets (compact / gap / improved / gap2...) + if tier in cfg and isinstance(cfg[tier], dict) and "prompts" in cfg[tier]: + return list(cfg[tier]["prompts"]) + + if tier == "all": + seen: set[str] = set() + out: list[str] = [] + for key in ( + "A_high", + "B_facility", + "C_detail", + "D_rail_domain", + "E_missed", + ): + block = cfg.get("tiers", {}).get(key, {}) + for p in block.get("prompts", []): + if p not in seen: + seen.add(p) + out.append(p) + # include scene_delta + for p in cfg.get("scene_delta", {}).get("prompts", []): + if p not in seen: + seen.add(p) + out.append(p) + return out + + if tier in cfg.get("tiers", {}): + return list(cfg["tiers"][tier]["prompts"]) + + # allow short aliases + aliases = { + "a": "A_high", + "A": "A_high", + "b": "B_facility", + "B": "B_facility", + "c": "C_detail", + "C": "C_detail", + "d": "D_rail_domain", + "D": "D_rail_domain", + "e": "E_missed", + "E": "E_missed", + "missed": "E_missed", + "gap_fill": "gap", + } + if tier in aliases: + key = aliases[tier] + if key in cfg.get("tiers", {}): + return list(cfg["tiers"][key]["prompts"]) + if key in cfg and "prompts" in cfg[key]: + return list(cfg[key]["prompts"]) + + raise ValueError( + f"Unknown tier '{tier}'. Use compact | gap | gap2 | improved | all | " + f"delta | A_high | B_facility | C_detail | D_rail_domain | E_missed | or --prompts" + ) + + +def resolve_image(path: Path | None, cfg: dict[str, Any]) -> Path: + candidates: list[Path] = [] + if path is not None: + candidates.append(path) + candidates.append(ROOT / cfg.get("image", "data/DJI_20260306100802_0016.JPG")) + candidates.append(DEFAULT_IMAGE) + candidates.append(SAMPLE_FALLBACK) + abs_hint = cfg.get("image_abs_hint") + if abs_hint: + candidates.append(Path(abs_hint)) + + for c in candidates: + if c and Path(c).is_file(): + return Path(c).resolve() + raise FileNotFoundError( + "Image not found. Place DJI JPG under data/ or pass --image. Tried:\n " + + "\n ".join(str(c) for c in candidates) + ) + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Multi-prompt SamGeo3 segmentation (Grok+Gemini prompt sets)" + ) + p.add_argument( + "--prompts-json", + type=Path, + default=DEFAULT_PROMPTS_JSON, + help="JSON with tiers/compact prompts", + ) + p.add_argument( + "--tier", + default="compact", + help=( + "compact | gap | gap2 | improved | all | " + "A_high | B_facility | C_detail | D_rail_domain | E_missed" + ), + ) + p.add_argument( + "--prompts", + default=None, + help='Comma-separated overrides, e.g. "building,tree,car"', + ) + p.add_argument("--image", type=Path, default=None) + p.add_argument("--model-id", default="facebook/sam3.1") + p.add_argument("--backend", default="meta", choices=["meta", "transformers"]) + p.add_argument("--confidence", type=float, default=0.3) + p.add_argument("--mask-threshold", type=float, default=0.5) + p.add_argument("--resolution", type=int, default=1008) + p.add_argument("--min-size", type=int, default=0) + p.add_argument("--max-size", type=int, default=None) + p.add_argument("--device", default=None) + p.add_argument( + "--checkpoint", + default=os.environ.get("SAM3_CHECKPOINT_PATH"), + ) + p.add_argument( + "--output-dir", + type=Path, + default=None, + help="Default: output/_/", + ) + p.add_argument("--no-viz", action="store_true") + p.add_argument( + "--skip-empty", + action="store_true", + default=True, + help="Do not write mask files when 0 objects (default true)", + ) + p.add_argument( + "--keep-empty", + action="store_true", + help="Opposite of --skip-empty", + ) + p.add_argument( + "--list-only", + action="store_true", + help="Print resolved prompts and exit (no model load)", + ) + return p.parse_args() + + +def main() -> int: + args = parse_args() + cfg_path = args.prompts_json.resolve() + if not cfg_path.is_file(): + print(f"ERROR: prompts json not found: {cfg_path}", file=sys.stderr) + return 2 + + cfg = load_prompt_config(cfg_path) + extra = None + if args.prompts: + extra = [x.strip() for x in args.prompts.split(",") if x.strip()] + + try: + prompts = resolve_prompts(cfg, args.tier, extra) + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 2 + + # de-dupe preserve order + seen: set[str] = set() + prompts = [p for p in prompts if not (p in seen or seen.add(p))] + + if args.list_only: + print(f"prompts_json: {cfg_path}") + print(f"tier: {args.tier if not extra else 'custom'}") + print(f"count: {len(prompts)}") + for i, pr in enumerate(prompts, 1): + print(f" {i:02d}. {pr}") + return 0 + + try: + image_path = resolve_image(args.image, cfg) + except FileNotFoundError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 2 + + if args.backend == "transformers" and "sam3.1" in args.model_id: + print( + "ERROR: facebook/sam3.1 requires backend='meta'.", + file=sys.stderr, + ) + return 2 + + tier_tag = "custom" if extra else args.tier + out_dir = ( + args.output_dir.resolve() + if args.output_dir + else (ROOT / "output" / f"{image_path.stem}_{tier_tag}").resolve() + ) + out_dir.mkdir(parents=True, exist_ok=True) + skip_empty = not args.keep_empty + + print("=== SamGeo3 multi-prompt segmentation ===") + print(f"image: {image_path}") + print(f"prompts_json:{cfg_path}") + print(f"tier: {tier_tag}") + print(f"n_prompts: {len(prompts)}") + print(f"model_id: {args.model_id}") + print(f"backend: {args.backend}") + print(f"confidence: {args.confidence}") + print(f"output_dir: {out_dir}") + print("prompts:") + for i, pr in enumerate(prompts, 1): + print(f" {i:02d}. {pr}") + + import numpy as np + import torch + from samgeo import SamGeo3 + + if args.device is None: + device = "cuda" if torch.cuda.is_available() else "cpu" + else: + device = args.device + + print(f"\ndevice: {device} | cuda={torch.cuda.is_available()}") + if device == "cuda" and not torch.cuda.is_available(): + print("ERROR: --device cuda but CUDA is not available", file=sys.stderr) + return 1 + + init_kwargs: dict[str, Any] = dict( + backend=args.backend, + model_id=args.model_id, + device=device, + confidence_threshold=args.confidence, + mask_threshold=args.mask_threshold, + resolution=args.resolution, + enable_segmentation=True, + enable_inst_interactivity=False, + ) + if args.checkpoint and os.path.isfile(args.checkpoint): + init_kwargs["checkpoint_path"] = args.checkpoint + init_kwargs["load_from_HF"] = False + print(f"checkpoint: {args.checkpoint}") + + t0 = time.perf_counter() + print("\nLoading model once...") + sam = SamGeo3(**init_kwargs) + print("set_image once...") + sam.set_image(str(image_path)) + t_load = time.perf_counter() - t0 + print(f"load+set_image: {t_load:.1f}s\n") + + results: list[dict[str, Any]] = [] + stem = image_path.stem + + for i, prompt in enumerate(prompts, 1): + t1 = time.perf_counter() + print(f"[{i}/{len(prompts)}] generate_masks({prompt!r})...") + gen_kwargs: dict[str, Any] = {"min_size": args.min_size, "quiet": True} + if args.max_size is not None: + gen_kwargs["max_size"] = args.max_size + + try: + sam.generate_masks(prompt, **gen_kwargs) + except Exception as e: + print(f" FAIL: {e}") + results.append( + { + "prompt": prompt, + "n_objects": 0, + "error": str(e), + "elapsed_s": round(time.perf_counter() - t1, 3), + } + ) + continue + + n = len(sam.masks) if getattr(sam, "masks", None) is not None else 0 + score_vals: list[float] = [] + scores = getattr(sam, "scores", None) + if scores is not None and len(scores): + try: + score_vals = [ + float(s.item() if hasattr(s, "item") else s) for s in scores + ] + except Exception: + pass + + rec: dict[str, Any] = { + "prompt": prompt, + "n_objects": n, + "scores_min": min(score_vals) if score_vals else None, + "scores_max": max(score_vals) if score_vals else None, + "scores_mean": float(np.mean(score_vals)) if score_vals else None, + "elapsed_s": round(time.perf_counter() - t1, 3), + "mask": None, + "ann": None, + "scores_npy": None, + } + + if n == 0: + print(f" -> 0 objects ({rec['elapsed_s']}s)") + results.append(rec) + continue + + print( + f" -> {n} objects | score " + f"{rec['scores_min']:.3f}~{rec['scores_max']:.3f} " + f"({rec['elapsed_s']}s)" + ) + + if skip_empty and n == 0: + results.append(rec) + continue + + tag = safe_name(prompt) + mask_path = out_dir / f"{stem}_{tag}_mask.png" + ann_path = out_dir / f"{stem}_{tag}_ann.png" + scores_path = out_dir / f"{stem}_{tag}_scores.npy" + + try: + sam.save_masks(str(mask_path), unique=True) + rec["mask"] = str(mask_path) + except Exception as e: + print(f" [WARN] save_masks: {e}") + + if score_vals: + np.save(str(scores_path), np.array(score_vals, dtype=np.float32)) + rec["scores_npy"] = str(scores_path) + + if not args.no_viz: + try: + sam.show_anns(output=str(ann_path)) + rec["ann"] = str(ann_path) + except Exception as e: + print(f" [WARN] show_anns: {e}") + + results.append(rec) + + summary = { + "image": str(image_path), + "prompts_json": str(cfg_path), + "tier": tier_tag, + "model_id": args.model_id, + "backend": args.backend, + "confidence": args.confidence, + "min_size": args.min_size, + "device": device, + "load_set_image_s": round(t_load, 3), + "total_s": round(time.perf_counter() - t0, 3), + "n_prompts": len(prompts), + "n_with_objects": sum(1 for r in results if r.get("n_objects", 0) > 0), + "results": results, + "sources": cfg.get("sources"), + "scene_notes_ko": cfg.get("scene_notes_ko"), + } + summary_path = out_dir / "summary.json" + with open(summary_path, "w", encoding="utf-8") as f: + json.dump(summary, f, ensure_ascii=False, indent=2) + + print("\n=== summary ===") + print(f"with objects: {summary['n_with_objects']}/{summary['n_prompts']}") + print(f"total time: {summary['total_s']}s") + print(f"summary: {summary_path}") + for r in results: + n = r.get("n_objects", 0) + mark = "OK" if n else "--" + print(f" [{mark}] {r['prompt']!r:30s} n={n}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/text_segment.py b/scripts/text_segment.py new file mode 100644 index 0000000..505b548 --- /dev/null +++ b/scripts/text_segment.py @@ -0,0 +1,189 @@ +"""Single-image text-prompt segmentation with SamGeo3 (meta backend). + +Usage (from project root, with venv active): + python scripts/text_segment.py + python scripts/text_segment.py --image data/test_image.jpg --prompt person + python scripts/text_segment.py --model-id facebook/sam3.1 --confidence 0.4 +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +# Project root = parent of scripts/ +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_DATA = ROOT / "data" +DEFAULT_OUTPUT = ROOT / "output" +SAMPLE_URL = ( + "https://raw.githubusercontent.com/facebookresearch/sam3/" + "refs/heads/main/assets/images/test_image.jpg" +) + + +def ensure_sample_image(path: Path) -> Path: + if path.is_file(): + return path + path.parent.mkdir(parents=True, exist_ok=True) + print(f"Downloading sample image -> {path}") + try: + from samgeo import download_file + + download_file(SAMPLE_URL, str(path)) + except Exception: + import urllib.request + + urllib.request.urlretrieve(SAMPLE_URL, str(path)) + if not path.is_file(): + raise FileNotFoundError(f"failed to obtain image: {path}") + return path + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="SamGeo3 text segmentation smoke test") + p.add_argument("--image", type=Path, default=DEFAULT_DATA / "test_image.jpg") + p.add_argument("--prompt", default="person", help="Text prompt for grounding") + p.add_argument("--model-id", default="facebook/sam3.1") + p.add_argument("--backend", default="meta", choices=["meta", "transformers"]) + # sam3.1 meta scores on this sample often peak ~0.3–0.4; 0.5 can yield zero masks + p.add_argument("--confidence", type=float, default=0.3) + p.add_argument("--mask-threshold", type=float, default=0.5) + p.add_argument("--resolution", type=int, default=1008) + p.add_argument("--min-size", type=int, default=0, help="Filter tiny masks (pixels)") + p.add_argument("--max-size", type=int, default=None) + p.add_argument("--device", default=None, help="cuda | cpu (default: auto)") + p.add_argument( + "--checkpoint", + default=os.environ.get("SAM3_CHECKPOINT_PATH"), + help="Local .pt path (or set SAM3_CHECKPOINT_PATH)", + ) + p.add_argument( + "--output-dir", + type=Path, + default=DEFAULT_OUTPUT, + help="Directory for mask / annotation outputs", + ) + p.add_argument( + "--no-viz", + action="store_true", + help="Skip matplotlib annotation PNG (mask file still saved)", + ) + return p.parse_args() + + +def main() -> int: + args = parse_args() + + if args.backend == "transformers" and "sam3.1" in args.model_id: + print( + "ERROR: facebook/sam3.1 requires backend='meta'. " + "Use model-id facebook/sam3 for transformers.", + file=sys.stderr, + ) + return 2 + + image_path = ensure_sample_image(args.image.resolve()) + out_dir = args.output_dir.resolve() + out_dir.mkdir(parents=True, exist_ok=True) + + stem = image_path.stem + safe_prompt = "".join(c if c.isalnum() or c in "-_" else "_" for c in args.prompt) + mask_path = out_dir / f"{stem}_{safe_prompt}_mask.png" + ann_path = out_dir / f"{stem}_{safe_prompt}_ann.png" + scores_path = out_dir / f"{stem}_{safe_prompt}_scores.npy" + + print("=== SamGeo3 text segmentation ===") + print(f"image: {image_path}") + print(f"prompt: {args.prompt}") + print(f"model_id: {args.model_id}") + print(f"backend: {args.backend}") + print(f"output_dir: {out_dir}") + + import torch + from samgeo import SamGeo3 + + if args.device is None: + device = "cuda" if torch.cuda.is_available() else "cpu" + else: + device = args.device + + print(f"device: {device} | cuda={torch.cuda.is_available()}") + if device == "cuda" and not torch.cuda.is_available(): + print("ERROR: --device cuda but CUDA is not available", file=sys.stderr) + return 1 + + init_kwargs = dict( + backend=args.backend, + model_id=args.model_id, + device=device, + confidence_threshold=args.confidence, + mask_threshold=args.mask_threshold, + resolution=args.resolution, + enable_segmentation=True, + enable_inst_interactivity=False, + ) + if args.checkpoint and os.path.isfile(args.checkpoint): + init_kwargs["checkpoint_path"] = args.checkpoint + init_kwargs["load_from_HF"] = False + print(f"checkpoint: {args.checkpoint}") + + print("\nLoading model...") + sam = SamGeo3(**init_kwargs) + + print("set_image...") + sam.set_image(str(image_path)) + + print(f'generate_masks("{args.prompt}")...') + gen_kwargs = {"min_size": args.min_size} + if args.max_size is not None: + gen_kwargs["max_size"] = args.max_size + sam.generate_masks(args.prompt, **gen_kwargs) + + n = len(sam.masks) if getattr(sam, "masks", None) is not None else 0 + if n == 0: + print("No masks found. Try another prompt or lower --confidence.") + return 0 + + scores = getattr(sam, "scores", None) + if scores is not None and len(scores): + try: + vals = [float(s.item() if hasattr(s, "item") else s) for s in scores] + print(f"scores (n={len(vals)}): min={min(vals):.3f} max={max(vals):.3f}") + except Exception: + pass + + print(f"Saving masks -> {mask_path}") + # PNG cannot store float score maps; save mask first, scores as .npy + sam.save_masks(str(mask_path), unique=True) + + scores = getattr(sam, "scores", None) + if scores is not None and len(scores): + import numpy as np + + score_vals = np.array( + [float(s.item() if hasattr(s, "item") else s) for s in scores], + dtype=np.float32, + ) + np.save(str(scores_path), score_vals) + print(f"Saved per-object scores -> {scores_path}") + + if not args.no_viz: + try: + print(f"Saving annotations -> {ann_path}") + sam.show_anns(output=str(ann_path)) + except Exception as e: + print(f"[WARN] show_anns failed: {e}") + + print(f"\nDone. Found {n} object(s).") + print(f" mask: {mask_path}") + if scores_path.is_file(): + print(f" scores: {scores_path}") + if ann_path.is_file(): + print(f" ann: {ann_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())