@inproceedings{kartchner-etal-2023-zero-shot,
title = "Zero-Shot Information Extraction for Clinical Meta-Analysis using Large Language Models",
author = "Kartchner, David  and
  Al-Hussaini, Ifran  and
  Ramalingam, Selvi  and
  Kronick, Olivia  and
  Mitchell, Cassie",
booktitle = "Proceedings of the 22nd Workshop on Biomedical Language Processing",
month = July,
year = "2023",
address = "Toronto, Canada",
publisher = "Association for Computational Linguistics",
doi = "10.18653/v1/2022.bionlp-1.1",
pages = "1--10",
abstract = "Meta-analysis of randomized clinical trials (RCTs) plays a crucial role in evidence-based medicine but can be labor-intensive and error-prone. This study explores the use of large language models to enhance the efficiency of aggregating results from randomized clinical trials (RCTs) at scale. We perform a detailed comparison of the performance of these models in zero-shot prompt-based information extraction from a diverse set of RCTs to traditional manual annotation methods. We analyze the results for two different meta-analyses aimed at drug repurposing in cancer therapy and pharmacovigilance in chronic myeloid leukemia. Our findings reveal that the best model for the two demonstrated tasks, ChatGPT, can generally extract correct information and identify when the desired information is missing from an article. We additionally conduct a systematic error analysis, documenting the prevalence of diverse error types encountered during the process of prompt-based information extraction. ",
}