{ "schema_version": "0.6.0", "paper_id": "picturing-ambiguity-winograd-schema", "page_url": "https://naser.github.io/research-publications/papers/picturing-ambiguity-winograd-schema/", "title": "Picturing Ambiguity: A Visual Twist on the Winograd Schema Challenge", "title_variants": [], "authors": [ "Brendan Park", "Madeline Janecek", "Naser Ezzati-Jivan", "Yifeng Li", "Ali Emami" ], "author_details": [ { "name": "Brendan Park", "orcid": null, "profile_url": "https://dblp.org/pid/316/0137.html" }, { "name": "Madeline Janecek", "orcid": null, "profile_url": "https://dblp.org/pid/307/5913.html" }, { "name": "Naser Ezzati-Jivan", "orcid": "https://orcid.org/0000-0003-1435-6297", "profile_url": "https://naser.github.io/" }, { "name": "Yifeng Li", "orcid": null, "profile_url": "https://dblp.org/pid/65/2432.html" }, { "name": "Ali Emami", "orcid": "https://orcid.org/0009-0001-4273-5203", "profile_url": "https://dblp.org/pid/75/10772.html" } ], "publication": { "year": 2024, "venue": "ACL 2024 Main Conference", "type": "conference paper", "publication_date": "2024", "online_date": null, "print_date": "2024", "volume": null, "issue": null, "pages": "355-374", "article_number": null, "publisher": "Association for Computational Linguistics", "issn": [], "isbn": [], "crossref_type": "proceedings-article" }, "publication_type": "conference paper", "status": "public_preprint_and_published_record", "canonical_source_url": "https://aclanthology.org/2024.acl-long.22/", "source_record_id": "picturing-ambiguity-a-visual-twist-on-the-winograd-schema-challenge-647b74a7c8", "identifiers": { "doi": "10.18653/v1/2024.acl-long.22", "arxiv": "2405.16277" }, "abstract": null, "abstract_source": "arXiv:2405.16277 abstract; public preprint version", "abstract_available": false, "scholar_eligibility": { "eligible": false, "basis": "not-eligible", "note": "The page is a discovery record; it does not claim Google Scholar article-host eligibility." }, "description": "The paper introduces WinoVis, a multimodal benchmark and analysis framework for testing pronoun disambiguation in text-to-image models.", "evidence_level": "full-text-reviewed", "evidence": { "source_basis": "full-text-reviewed", "coverage": "material paper sections", "summary_origin": "AI-assisted catalog editorial summary", "review_status": "catalog-reviewed; paper-author approval pending", "verified_on": "2026-08-09", "sources": [ { "note": "ACL Anthology and DOI metadata" }, { "note": "official arXiv abstract and PDF page 2405.16277" }, { "note": "local PDF hash verified in the working catalog" } ] }, "summary": { "core_contribution": "The paper introduces WinoVis, a multimodal benchmark and analysis framework for testing pronoun disambiguation in text-to-image models.", "problem": "Text-to-image models can generate visually plausible images without resolving which entity an ambiguous pronoun refers to. Existing WSC-style reasoning evaluations are mainly textual and do not isolate multimodal pronoun disambiguation from image-generation artifacts.", "method": "WINOVIS contains 500 WSC-adapted scenarios generated through GPT-4 prompting and manually filtered for textual ambiguity, illogical content, visual indistinctiveness, and redundancy. The evaluation uses Stable Diffusion generations, DAAM cross-attention heatmaps, caption filtering, 90th-percentile heatmap thresholding, and IoU-based overlap/decision rules. A single pronoun-to-entity association is accepted when the pronoun heatmap crosses IoU 0.4 with one or the stronger of the two referents.", "findings": "The paper evaluates Stable Diffusion versions labeled 1.0/1.5/2.0 and XL in Tables 2-3; the setup prose says 1.1/1.5/2.0/XL. SD 2.0 reports 56.7% precision, 24.2% recall, 34.1% F1, and 36.1% certainty, with 55 correct, 42 incorrect, and 172 neither outcomes in the table. SDXL produces mostly unusable heatmap decisions. Error analysis shows much weaker handling of visually distinct entities than disparate entities.", "limitations": "Entity separation, semantic entanglement, captioned images, DAAM availability only for open Stable Diffusion models, bias, and limited scenario diversity affect validity. The metrics assess heatmap/association behavior rather than human-level or task-functional multimodal reasoning. GPU/CPU, OS, runtime, and replication details are unknown.", "future_work": "Improve entity separation and entanglement filtering, study bias, expand context and entity diversity, and develop interpretability methods for more diffusion models." }, "tags": [ "multimodal-ai", "benchmark-datasets", "common-sense-reasoning", "machine-learning" ], "keywords": [ "Winograd Schema Challenge", "WinoVis", "text-to-image models", "pronoun disambiguation", "DAAM", "Stable Diffusion" ], "versions": [ { "id": "published-version", "label": "Published version", "relation": "version-of-record", "title": "Picturing Ambiguity: A Visual Twist on the Winograd Schema Challenge", "url": "https://aclanthology.org/2024.acl-long.22/", "pdf_url": null, "status": "published_public_page", "canonical_for_citation": true }, { "id": "public-preprint", "label": "Public preprint", "relation": "preprint", "title": "Picturing Ambiguity: A Visual Twist on the Winograd Schema Challenge", "url": "https://arxiv.org/abs/2405.16277", "pdf_url": "https://arxiv.org/pdf/2405.16277", "status": "public_full_text", "canonical_for_citation": false } ], "access": { "status": "public_preprint_and_published_record", "note": "The ACL DOI is the canonical citation target; the arXiv preprint and ACL Anthology page provide public access paths.", "license": null }, "resources": { "code": null, "data": null, "slides": null, "demo": null }, "citation_guidance": { "when_to_cite": "Cite this paper when your work uses or compares wINOVIS as a 500-scenario benchmark for pronoun disambiguation in text-to-image generation.", "points": [ "WINOVIS as a 500-scenario benchmark for pronoun disambiguation in text-to-image generation.", "a DAAM heatmap plus IoU evaluation pipeline that separates entity overlap, caption artifacts, and pronoun-association decisions.", "the SD 2.0 result when documenting that visually convincing diffusion outputs can still show weak multimodal pronoun resolution.", "the distinct-versus-disparate entity error analysis when motivating harder visual common-sense benchmarks." ], "canonical_version_id": "published-version" }, "provenance": { "metadata_verified_on": "2026-08-09", "metadata_source": [ "ACL Anthology and DOI metadata", "official arXiv abstract and PDF page 2405.16277", "local PDF hash verified in the working catalog" ], "summary_written_by": "AI-assisted", "summary_verified_by": "full-text-grounded catalog review; author approval pending", "linked_preprint_record": null, "author_order_note": null }, "batch": { "phase": 2, "batch_label": "expanded forty-paper release", "status": "included_in_expanded_catalog", "selected_at": "2026-08-09" } }