@article{196,
  author = {Kiran Vodrahalli and Po-Hsuan Chen and Yingyu Liang and Christopher Baldassano and Janice Chen and Esther Young and Christopher Honey and Uri Hasson and Peter Ramadge and Kenneth Norman and Sanjeev Arora},
  title = {Mapping Between fMRI Responses to Movies and their Natural Language Annotations},
  abstract = {<p>Several research groups have shown how to map fMRI responses to the meanings of presented stimuli. This paper presents new methods for doing so when only a natural language annotation is available as the description of the stimulus. We study fMRI data gathered from subjects watching an episode of BBCs Sherlock (<a href="https://www.sciencedirect.com/science/article/pii/S1053811917305128$\#$bib6">Chen et\&nbsp;al., 2017</a>), and learn bidirectional mappings between fMRI responses and natural language representations. By leveraging data from multiple subjects watching the same movie, we were able to perform scene classification with 72\% accuracy (random guessing would give 4\%) and scene ranking with average rank in the top 4\% (random guessing would give 50\%). The key ingredients underlying this high level of performance are (a) the use of the Shared Response Model (SRM) and its variant SRM-ICA (<a href="https://www.sciencedirect.com/science/article/pii/S1053811917305128$\#$bib7">Chen et\&nbsp;al., 2015</a>,\&nbsp;<a href="https://www.sciencedirect.com/science/article/pii/S1053811917305128$\#$bib25">Zhang et\&nbsp;al., 2016</a>) to aggregate fMRI data from multiple subjects, both of which are shown to be superior to standard PCA in producing low-dimensional representations for the tasks in this paper; (b) a sentence embedding technique adapted from the\&nbsp;<a href="https://www.sciencedirect.com/topics/neuroscience/natural-language-processing">natural language processing</a>\&nbsp;(NLP) literature (<a href="https://www.sciencedirect.com/science/article/pii/S1053811917305128$\#$bib4">Arora et\&nbsp;al., 2017</a>) that produces semantic vector representation of the annotations; (c) using previous timestep information in the featurization of the predictor data. These optimizations in how we featurize the fMRI data and text annotations provide a substantial improvement in classification performance, relative to standard approaches.</p>
},
  year = {2018},
  journal = {NeuroImage},
  volume = {180},
  pages = {223-231},
  doi = {10.1016/j.neuroimage.2017.06.042},
}