FR-045 CMMC 2.0 v2.13 (2024) PDF Parser 相容性修正 — Implementation Plan

git -C /Users/chouraymond/Projects/Jedicogy/module/jedi-python-package/jedi-oscal-v2 status --short
git -C /Users/chouraymond/Projects/Jedicogy/module/jedi-python-package/jedi-oscal-v2 branch --show-current

cd /Users/chouraymond/Projects/Billows/Audit-Manager/compliance-manager-be
poetry run python -c "import jedi_oscal_v2, os; print(os.path.realpath(jedi_oscal_v2.__file__))"

# baseline_dump.py — 修正前基準
import json
from jedi_oscal_v2.infra.adapter.cmmc.cmmc2_lv2_parser_adapter import CMMC2Level2Adapter

FILES = {
    "old_l1": "/Users/chouraymond/Desktop/00_Compliance Manager/稽核法規文件/CMMC 2.0/CMMC 2.0 Level 1 Assessment Guide.pdf",
    "old_l2": "/Users/chouraymond/Desktop/00_Compliance Manager/稽核法規文件/CMMC 2.0/CMMC 2.0 Level 2 Assessment Guide.pdf",
}
a = CMMC2Level2Adapter()
for tag, f in FILES.items():
    with open(f, "rb") as fh:
        recs = a.pdf_parser(fh)
    with open(f"/tmp/cmmc_baseline_{tag}.json", "w") as out:
        json.dump(recs, out, ensure_ascii=False, indent=1, sort_keys=True)
    print(tag, len(recs))

# font_check.py — throwaway
import pdfplumber, re
for f in [
    "/Users/chouraymond/Desktop/00_Compliance Manager/稽核法規文件/CMMC 2.0/CMMC 2.0 Level 2 Assessment Guide.pdf",
    "/Users/chouraymond/Desktop/00_Compliance Manager/稽核法規文件/CMMC 2.0/CMMC 2.0 Level 2 Assessment Guide v2.pdf",
]:
    with pdfplumber.open(f) as pdf:
        for p in pdf.pages[40:46]:
            fonts = {w["fontname"].split("+")[-1] for w in p.extract_words(extra_attrs=["fontname"])}
            print(f.split("/")[-1], sorted(fonts))
            break

# -*- coding: utf-8 -*-
"""FR-045 — CMMC v2.13 (2024) guide 相容：regex 雙格式 / ID 正規化 / 標題尾綴 / 字型分流組行。

純 unit（合成資料）；四份真實 PDF 的 E2E 驗證見 implementation-plan Task 5。
"""
import pytest

from jedi_oscal_v2.infra.adapter.cmmc.cmmc2_lv2_parser_adapter import (
    _PRACTICE_PATTERN,
    _NEXT_PRACTICE_ID_PATTERN,
    _TITLE_DATA_SUFFIX,
    CMMC2Level2Adapter,
    _normalize_practice_id,
)


class TestPracticePattern:
    @pytest.mark.parametrize(
        "line,pid,title",
        [
            # 2021 NIST 式（L1 / L2）
            ("AC.L1-3.1.1 – Authorized Access Control", "AC.L1-3.1.1", "Authorized Access Control"),
            ("AC.L2-3.1.20 – External Connections", "AC.L2-3.1.20", "External Connections"),
            # 2024 FAR 式：本文標題全大寫 + 尾綴
            (
                "AC.L1-B.1.I – AUTHORIZED ACCESS CONTROL [FCI DATA]",
                "AC.L1-B.1.I",
                "AUTHORIZED ACCESS CONTROL [FCI DATA]",
            ),
            # 2024 FAR 式：小寫變體（目錄 / 頁眉樣式）
            (
                "SI.L1-b.1.xii – Flaw Remediation [FCI Data]",
                "SI.L1-b.1.xii",
                "Flaw Remediation [FCI Data]",
            ),
        ],
    )
    def test_matches_both_numbering_schemes(self, line, pid, title):
        m = _PRACTICE_PATTERN.match(line)
        assert m, f"should match: {line}"
        assert m.group(1) == pid
        assert m.group(2) == title

    @pytest.mark.parametrize(
        "line",
        [
            "Internet Protocol (IP)",                    # 一般散文
            "see AC.L1-3.1.1 for more information",      # 行中引用（非行首）
            "AC.L1-3.1.1 no separator title",            # 缺 – 分隔
            "Access Control (AC)",                        # domain 標題
        ],
    )
    def test_rejects_non_practice_lines(self, line):
        assert _PRACTICE_PATTERN.match(line) is None

    def test_next_practice_pattern_accepts_far_style(self):
        assert _NEXT_PRACTICE_ID_PATTERN.match("IA.L1-b.1.v additional text")
        assert _NEXT_PRACTICE_ID_PATTERN.match("IA.L1-3.5.1 additional text")


class TestNormalizePracticeId:
    @pytest.mark.parametrize(
        "raw,expected",
        [
            ("AC.L1-B.1.I", "AC.L1-b.1.i"),       # 全大寫本文標題 → 官方小寫制式
            ("AC.L1-b.1.iv", "AC.L1-b.1.iv"),     # 已是小寫 → 不變
            ("AC.L2-3.1.1", "AC.L2-3.1.1"),       # NIST 式 → 不變
            ("SI.L1-B.1.XV", "SI.L1-b.1.xv"),
        ],
    )
    def test_normalize(self, raw, expected):
        assert _normalize_practice_id(raw) == expected


class TestTitleDataSuffix:
    @pytest.mark.parametrize(
        "raw,expected",
        [
            ("AUTHORIZED ACCESS CONTROL [FCI DATA]", "AUTHORIZED ACCESS CONTROL"),
            ("Authorized Access Control [CUI Data]", "Authorized Access Control"),
            ("External Connections", "External Connections"),  # 無尾綴不變
        ],
    )
    def test_strip(self, raw, expected):
        assert _TITLE_DATA_SUFFIX.sub("", raw).strip() == expected

cd /Users/chouraymond/Projects/Billows/Audit-Manager/compliance-manager-be
poetry run pytest /Users/chouraymond/Projects/Jedicogy/module/jedi-python-package/jedi-oscal-v2/tests/catalog/test_cmmc_pdf_parser_v213_compat.py -v

# Practice-ID 雙格式（FR-045）：
#   2021 guides（NIST SP 800-171 式）:      AC.L1-3.1.1 / AC.L2-3.1.1
#   2024 v2.13 guides（FAR 52.204-21 式）:  AC.L1-b.1.i（羅馬數字；本文標題為全大寫 AC.L1-B.1.I）
_PRACTICE_ID_CORE = r"[A-Z]{2}\.L\d-(?:\d+\.\d+\.\d+|[A-Za-z]\.\d+\.[IVXivx]+)"
_PRACTICE_PATTERN = re.compile(rf"^({_PRACTICE_ID_CORE})\s*[-–—]\s*(.+)$")
_PRACTICE_CODE_PATTERN = re.compile(r"^([A-Z]{2})\.L\d-")
_NEXT_PRACTICE_ID_PATTERN = re.compile(rf"^({_PRACTICE_ID_CORE})\s*")
# 2024 版標題尾綴 [FCI Data] / [CUI DATA]（大小寫不拘）
_TITLE_DATA_SUFFIX = re.compile(r"\s*\[\s*(?:FCI|CUI)\s+DATA\s*\]\s*$", re.IGNORECASE)
_FAR_ID_NORMALIZE = re.compile(r"^([A-Z]{2}\.L\d)-([A-Za-z])\.(\d+)\.([IVXivx]+)$")


def _normalize_practice_id(pid: str) -> str:
    """FAR 式 ID 正規化成官方小寫制式（AC.L1-B.1.I → AC.L1-b.1.i）；NIST 式原樣返回。"""
    m = _FAR_ID_NORMALIZE.match(pid)
    if not m:
        return pid
    return f"{m.group(1)}-{m.group(2).lower()}.{m.group(3)}.{m.group(4).lower()}"

        # -------- Regex 定義 --------
        domain_pattern = re.compile(r"^(.+)\s+\(([A-Z]{2})\)$")
        practice_pattern = _PRACTICE_PATTERN
        practice_code_pattern = _PRACTICE_CODE_PATTERN
        ao_start_pattern = re.compile(r"^\[([a-z])\]\s*(.*)$")
        toc_pattern = re.compile(r".{5,}\s\d{1,3}$")
        dots_pattern = re.compile(r"\.{5,}")

                        pid = _normalize_practice_id(m.group(1))
                        pname = _TITLE_DATA_SUFFIX.sub("", m.group(2)).strip()

poetry run pytest /Users/chouraymond/Projects/Jedicogy/module/jedi-python-package/jedi-oscal-v2/tests/catalog/test_cmmc_pdf_parser_v213_compat.py -v

poetry run pytest /Users/chouraymond/Projects/Jedicogy/module/jedi-python-package/jedi-oscal-v2/tests/catalog/ -v

cd /Users/chouraymond/Projects/Jedicogy/module/jedi-python-package/jedi-oscal-v2
git add jedi_oscal_v2/infra/adapter/cmmc/cmmc2_lv2_parser_adapter.py tests/catalog/test_cmmc_pdf_parser_v213_compat.py
git commit -m "feat(FR-045): CMMC practice regex 雙格式 — 2024 v2.13 FAR 式編號 (AC.L1-b.1.i) + [FCI/CUI Data] 標題尾綴剝除"

def _w(text, top, font="XDXQPJ+Cambria"):
    """合成 pdfplumber word dict（只含 parser 用到的 key）。"""
    return {"text": text, "top": top, "fontname": font}


_BOLD = "MFJSRF+Cambria-Bold"


class TestMergeFontStreams:
    def test_plain_lines_unchanged(self):
        """單一字型、逐行遞增 y — 行為與舊版逐 y 組行一致。"""
        words = [
            _w("Limit", 100.0), _w("system", 100.0), _w("access", 100.0),
            _w("Determine", 120.0), _w("if:", 120.0),
        ]
        assert CMMC2Level2Adapter._merge_font_streams(words) == [
            "Limit system access",
            "Determine if:",
        ]

    def test_bold_heading_near_same_y_becomes_own_line(self):
        """v2.13 疊字場景：Bold 標題與內文 y 差 0.2 → 必須拆成獨立兩行，內文在前。"""
        words = [
            _w("or", 193.2), _w("devices", 193.2),
            _w("ASSESSMENT", 193.4, _BOLD), _w("OBJECTIVES", 193.4, _BOLD),
            _w("Determine", 249.2), _w("if:", 249.2),
        ]
        assert CMMC2Level2Adapter._merge_font_streams(words) == [
            "or devices",
            "ASSESSMENT OBJECTIVES",
            "Determine if:",
        ]

    def test_equal_y_body_sorts_before_bold(self):
        """完全同 y（實測 369.6 == 369.6）：內文行必須排在 Bold 標題前，
        否則收 AO 的最後一條（[f]）會被標題先終結而遺失。"""
        words = [
            _w("[f]", 369.6), _w("system", 369.6), _w("access", 369.6),
            _w("POTENTIAL", 369.6, _BOLD), _w("METHODS", 369.6, _BOLD),
        ]
        assert CMMC2Level2Adapter._merge_font_streams(words) == [
            "[f] system access",
            "POTENTIAL METHODS",
        ]

    def test_header_words_above_threshold_dropped(self):
        words = [_w("RunningHeader", 30.0), _w("body", 100.0)]
        assert CMMC2Level2Adapter._merge_font_streams(words) == ["body"]

    def test_bold_lines_grouped_by_y_within_stream(self):
        """Bold stream 自己也要逐 y 組行（Examine 與 Interview 是不同標題行）。"""
        words = [
            _w("Examine", 399.5, _BOLD),
            _w("Interview", 558.2, _BOLD),
        ]
        assert CMMC2Level2Adapter._merge_font_streams(words) == ["Examine", "Interview"]

poetry run pytest /Users/chouraymond/Projects/Jedicogy/module/jedi-python-package/jedi-oscal-v2/tests/catalog/test_cmmc_pdf_parser_v213_compat.py -v -k MergeFontStreams

    @staticmethod
    def _merge_font_streams(
        words: list[dict], header_y_threshold: float = 50, line_gap: float = 8
    ) -> list[str]:
        """把 words 依字型分成 Bold（章節標題）/ 一般（內文）兩條 stream，
        各自沿用「y 跳距 > line_gap 斷行」的組行邏輯，最後按 (y, is_bold)
        排序合併回閱讀序。

        動機（FR-045）：2024 v2.13 CMMC guide 的 PDF 把粗體章節標題
        （ASSESSMENT OBJECTIVES / POTENTIAL ASSESSMENT METHODS AND OBJECTS /
        Examine / Interview / Test）排在與內文行相同的 y 座標，單一 stream
        組行會把標題併進內文行，導致區塊切分失效（AO 遺失、methods 全進
        examine 桶）。2021 版標題本來就獨立成行，分流後輸出不變。

        同 y 時內文排在 Bold 前（is_bold False < True）：AO 末條與
        POTENTIAL... 標題同 y 時，必須先收完 AO 再讓標題終結區塊。
        """
        streams: dict[bool, list[dict]] = {False: [], True: []}
        for w in words:
            if float(w["top"]) < header_y_threshold:
                continue
            streams["Bold" in w.get("fontname", "")].append(w)

        lines: list[tuple[float, bool, str]] = []
        for is_bold, stream_words in streams.items():
            current: list[str] = []
            anchor_y = None
            for w in stream_words:
                y = float(w["top"])
                if anchor_y is None:
                    anchor_y = y
                if abs(y - anchor_y) > line_gap:
                    if current:
                        lines.append((anchor_y, is_bold, " ".join(current)))
                        current = []
                    anchor_y = y
                current.append(w["text"])
            if current:
                lines.append((anchor_y, is_bold, " ".join(current)))

        lines.sort(key=lambda t: (t[0], t[1]))
        return [text for _, _, text in lines]

        def _extract_lines_from_page(p):
            words = p.extract_words(use_text_flow=True, extra_attrs=["fontname"])
            return CMMC2Level2Adapter._merge_font_streams(
                words, header_y_threshold=header_y_threshold
            )

poetry run pytest /Users/chouraymond/Projects/Jedicogy/module/jedi-python-package/jedi-oscal-v2/tests/catalog/ -v

cd /Users/chouraymond/Projects/Jedicogy/module/jedi-python-package/jedi-oscal-v2
git add jedi_oscal_v2/infra/adapter/cmmc/cmmc2_lv2_parser_adapter.py tests/catalog/test_cmmc_pdf_parser_v213_compat.py
git commit -m "fix(FR-045): PDF 行抽取字型分流 — v2.13 guide 粗體標題與內文同 y 疊字導致 AO 遺失 / methods 分桶全毀"

# verify_all.py — FR-045 E2E 驗收
import json
from jedi_oscal_v2.infra.adapter.cmmc.cmmc2_lv2_parser_adapter import CMMC2Level2Adapter

BASE = "/Users/chouraymond/Desktop/00_Compliance Manager/稽核法規文件/CMMC 2.0"
FILES = {
    "old_l1": f"{BASE}/CMMC 2.0 Level 1 Assessment Guide.pdf",
    "new_l1": f"{BASE}/CMMC 2.0 Level 1 Assessment Guide v2.pdf",
    "old_l2": f"{BASE}/CMMC 2.0 Level 2 Assessment Guide.pdf",
    "new_l2": f"{BASE}/CMMC 2.0 Level 2 Assessment Guide v2.pdf",
}
a = CMMC2Level2Adapter()
parsed = {}
for tag, f in FILES.items():
    with open(f, "rb") as fh:
        recs = a.pdf_parser(fh)
    parsed[tag] = recs
    ao = sum(len(r["assessment_objectives"]) for r in recs)
    buckets = {k: 0 for k in ("examine", "interview", "test")}
    for r in recs:
        for k, v in r["methods"].items():
            buckets[k] += len(v)
    zero_ao = [r["practice_id"] for r in recs if not r["assessment_objectives"]]
    zero_m = [r["practice_id"] for r in recs if not any(r["methods"].values())]
    print(f"{tag}: practices={len(recs)} AOs={ao} buckets={buckets} zero_AO={zero_ao} zero_methods={zero_m}")

# 舊版兩檔與修正前基準逐欄 diff（AO / methods 必須逐條一致；description 允許輕微差異需人工看過）
for tag in ("old_l1", "old_l2"):
    with open(f"/tmp/cmmc_baseline_{tag}.json") as fh:
        base = {r["practice_id"]: r for r in json.load(fh)}
    now = {r["practice_id"]: r for r in parsed[tag]}
    assert set(base) == set(now), f"{tag} practice id set changed!"
    for pid in base:
        assert base[pid]["assessment_objectives"] == now[pid]["assessment_objectives"], f"{tag} {pid} AO diff"
        assert base[pid]["methods"] == now[pid]["methods"], f"{tag} {pid} methods diff"
        if base[pid]["description"] != now[pid]["description"]:
            print(f"[description diff — 人工確認] {tag} {pid}")
    print(f"{tag}: baseline diff OK")

# 新 L1 spot-check：官方 AC.L1-b.1.i 有 [a]~[f] 六條 AO
l1 = {r["practice_id"]: r for r in parsed["new_l1"]}
assert len(l1["AC.L1-b.1.i"]["assessment_objectives"]) == 6, l1["AC.L1-b.1.i"]["assessment_objectives"]
print("new_l1 spot-check OK")

# 新版 description 抽查（reviewer 指出的盲點：字型分流若把內文 inline 粗體字
# 抽成獨立行，description 會被重排 — 數量統計驗不出來，要人工對 PDF 原文眼球比對）
for tag, pids in (("new_l1", ["AC.L1-b.1.i", "SI.L1-b.1.xii"]), ("new_l2", ["AC.L2-3.1.1", "CM.L2-3.4.4", "SC.L2-3.13.11"])):
    d = {r["practice_id"]: r for r in parsed[tag]}
    for pid in pids:
        print(f"[description 抽查 — 對 PDF 原文眼球比對] {tag} {pid}: {d[pid]['description'][:300]}")