"""Resolve CertyIQ / PDF question numbers from marker placement.""" from __future__ import annotations import re MARKER_Q = re.compile( r"<<>>|<<>>|Question:\s*(\d+)", re.I, ) PREFACE_ID_BASE = 90000 CASE_STUDY_INTRO = "After you answer a question in this section" ANSWER_LINE = re.compile(r"\bAnswer:\s*", re.I) def parse_markers(text: str) -> list[int]: ids: list[int] = [] for m in MARKER_Q.finditer(text or ""): ids.append(int(m.group(1) or m.group(2) or m.group(3))) return ids NEXT_QUESTION = re.compile( r"\n(?:" r"Your company |You have |You need |You want |You plan |You are |You must |" r"Which |What should |What is |What are |How |Each |An administrator|Contoso|Given |" r"HOTSPOT|DRAG DROP|For each of the following|Does the solution|" r"A company|There is an? |After you|If you" r")", re.I, ) def infer_pdf_id_from_text(text: str) -> int | None: """ Resolve PDF question id from CertyIQ markers. CertyIQ prints `Question: N` after Answer/Explanation (often mid-explanation on page break). Use the **last** `Answer:` in the span, then the **last** `Question:N` before the next question stem. """ if not text: return None answers = list(ANSWER_LINE.finditer(text)) if answers: after_text = text[answers[-1].end() :] nxt = NEXT_QUESTION.search(after_text) if nxt: after_text = after_text[: nxt.start()] after_markers = parse_markers(after_text) if after_markers: return after_markers[-1] before_text = text[: answers[-1].start()] before_markers = parse_markers(before_text) if before_markers: return before_markers[-1] markers = parse_markers(text) return markers[-1] if markers else None def _case_study_runs(review: list[dict]) -> list[list[dict]]: runs: list[list[dict]] = [] run: list[dict] = [] for item in review: if CASE_STUDY_INTRO in (item.get("questionText") or ""): run.append(item) else: if run: runs.append(run) run = [] if run: runs.append(run) return runs def _assign_case_study_run( run: list[dict], used: set[int], certyiq_base: str ) -> None: """Sequential Question:N within one case-study block (407, 408, 409, ...).""" if not run: return first = run[0] start = first.get("pdfId") if start is None or int(start) >= PREFACE_ID_BASE: gap = first.get("markersInGap") or [] start = gap[0] if gap else None if start is None or int(start) >= PREFACE_ID_BASE: exp = parse_markers(first.get("explanation")) start = exp[0] if len(exp) == 1 else None if len(run) == 1: item = run[0] exp = parse_markers(item.get("explanation")) pid = item.get("pdfId") seq = int(item.get("seq") or 0) if len(exp) == 1: doc_id = exp[0] elif ( pid and int(pid) < PREFACE_ID_BASE and seq > 100 and int(pid) < seq and (seq - int(pid)) <= 5 ): doc_id = seq elif start is not None and int(start) < PREFACE_ID_BASE: doc_id = int(start) elif seq > 2: doc_id = seq else: doc_id = int(pid) if pid else seq _set_pdf_id(item, doc_id, used, certyiq_base) return if start is None or int(start) >= PREFACE_ID_BASE: seq0 = int(first.get("seq") or 0) start = seq0 if seq0 > 2 else 1 start = int(start) for k, item in enumerate(run): doc_id = start + k while doc_id in used: doc_id += 1 _set_pdf_id(item, doc_id, used, certyiq_base) def review_item_quality(item: dict) -> int: score = len(item.get("questionText") or "") stem = (item.get("stem") or "").strip() if stem: score += 200 opts = item.get("options") or [] if opts: score += 400 keys = (item.get("answer") or {}).get("keys") or [] if keys: score += 300 if item.get("explanation"): score += min(len(item["explanation"]), 800) return score def merge_review_items(primary: dict, secondary: dict) -> dict: """Fill missing fields in `primary` from `secondary`.""" if review_item_quality(secondary) > review_item_quality(primary): primary, secondary = secondary, primary if not (primary.get("questionText") or "").strip() and secondary.get("questionText"): primary["questionText"] = secondary["questionText"] primary["stem"] = secondary.get("stem") or primary.get("stem") primary["options"] = secondary.get("options") or primary.get("options") primary["type"] = secondary.get("type") or primary.get("type") if not (primary.get("answer") or {}).get("keys") and (secondary.get("answer") or {}).get("keys"): primary["answer"] = secondary["answer"] sec_exp = secondary.get("explanation") or "" pri_exp = primary.get("explanation") or "" if len(sec_exp) > len(pri_exp): primary["explanation"] = sec_exp for field in ("reference", "note", "references"): if not primary.get(field) and secondary.get(field): primary[field] = secondary[field] try: from extract_pdf_review import repair_block_from_marker_text repair_block_from_marker_text(primary) except Exception: pass flags = primary.setdefault("flags", {}) sec_flags = secondary.get("flags") or {} for key, val in sec_flags.items(): if key.startswith("missing") and val is False: flags[key] = False return primary def dedupe_review_blocks(blocks: list[dict]) -> list[dict]: by_id: dict[int, dict] = {} orphans: list[dict] = [] for item in blocks: pid = item.get("pdfId") if pid is None: orphans.append(item) continue pid = int(pid) if pid in by_id: by_id[pid] = merge_review_items(by_id[pid], item) else: by_id[pid] = item merged = [by_id[k] for k in sorted(by_id)] merged.extend(orphans) return merged def _set_pdf_id( item: dict, doc_id: int, used: set[int], certyiq_base: str ) -> None: item["pdfId"] = doc_id item.setdefault("source", {})["certyiqUrl"] = f"{certyiq_base}/{doc_id}" if item.get("flags"): item["flags"]["missingPdfId"] = False used.add(doc_id) def assign_and_deduplicate_review_blocks( blocks: list[dict], *, certyiq_base: str, preface_seq_max: int = 0, ) -> list[dict]: """Assign pdfId from Question:N markers, then merge duplicate ids.""" used: set[int] = set() for run in _case_study_runs(blocks): _assign_case_study_run(run, used, certyiq_base) for item in blocks: if ( CASE_STUDY_INTRO in (item.get("questionText") or "") and item.get("pdfId") is not None ): continue marker_text = item.get("markerText") or item.get("questionText") or "" inferred = infer_pdf_id_from_text(marker_text) seq = int(item.get("seq") or 0) if inferred is not None: doc_id = inferred elif seq <= preface_seq_max: doc_id = PREFACE_ID_BASE + seq elif seq > 0: doc_id = seq else: continue item["pdfId"] = doc_id item.setdefault("source", {})["certyiqUrl"] = f"{certyiq_base}/{doc_id}" if item.get("flags"): item["flags"]["missingPdfId"] = False return dedupe_review_blocks(blocks) def assign_document_ids( review: list[dict], *, certyiq_base: str = "https://certyiq.com/papers/microsoft/az-104", preface_seq_max: int = 2, ) -> dict[int, int]: """Backward-compatible wrapper.""" before = {int(b["pdfId"]) for b in review if b.get("pdfId") is not None} merged = assign_and_deduplicate_review_blocks( review, certyiq_base=certyiq_base, preface_seq_max=preface_seq_max, ) review[:] = merged after = {int(b["pdfId"]) for b in review if b.get("pdfId") is not None} return {} def is_preface_id(qid: int) -> bool: return qid in (90001, 90002) def is_junk_id(qid: int) -> bool: """Ids from collision fallback (90000 + seq); not real PDF question numbers.""" return qid >= PREFACE_ID_BASE and not is_preface_id(qid)