@InProceedings{neurips19diplomacy,
  author    = {Philip Paquette, Yuchen Lu, Steven Bocco, Max O. Smith, Satya Ortiz-Gagn{\'e}, Jonathan K. Kummerfeld, Joelle Pineau, Satinder Singh, Aaron Courville},
  title     = {No-Press Diplomacy: Modeling Multi-Agent Gameplay},
  booktitle = {Advances in Neural Information Processing Systems 32},
  year      = {2019},
  month     = {December},
  pages     = {},
  url       = {},
  arxiv     = {https://arxiv.org/abs/1909.02128},
}

      

Diplomacy is a seven-player non-stochastic, non-cooperative game, where agents acquire resources through a mix of teamwork and betrayal. Reliance on trust and coordination makes Diplomacy the first non-cooperative multi-agent benchmark for complex sequential social dilemmas in a rich environment. In this work, we focus on training an agent that learns to play the No Press version of Diplomacy where there is no dedicated communication channel between players. We present DipNet, a neural-network-based policy model for No Press Diplomacy. The model was trained on a new dataset of more than 150,000 human games. Our model is trained by supervised learning (SL) from expert trajectories, which is then used to initialize a reinforcement learning (RL) agent trained through self-play. Both the SL and RL agents demonstrate state-of-the-art No Press performance by beating popular rule-based bots.

@InProceedings{emnlp19data,
  title     = {An Evaluation for Intent Classification and Out-of-Scope Prediction},
  author    = {Stefan Larson, Anish Mahendran, Joseph J. Peper, Christopher Clarke, Andrew Lee, Parker Hill, Jonathan K. Kummerfeld, Kevin Leach, Michael A. Laurenzano, Lingjia Tang and Jason Mars},
  booktitle = {Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing},
  month     = {November},
  year      = {2019},
  location  = {Hong Kong, China},
  pages     = {},
  url       = {},
  data      = {https://github.com/clinc/oos-eval},
  arxiv     = {https://arxiv.org/abs/1909.02027},
}

      

Task-oriented dialog systems need to know when a query falls outside their range of supported intents, but current text classification corpora only define label sets that cover every example. We introduce a new dataset that includes queries that are out-of-scope—i.e., queries that do not fall into any of the system’s supported intents. This poses a new challenge because models cannot assume that every query at inference time belongs to a system-supported intent class. Our dataset also covers 150 intent classes over 10 domains, capturing the breadth that a production task-oriented agent must handle. We evaluate a range of benchmark classifiers on our dataset along with several different out-of-scope identification schemes. We find that while the classifiers perform well on in-scope intent classification, they struggle to identify out-of-scope queries. Our dataset and evaluation fill an important gap in the field, offering a way of more rigorously and realistically benchmarking text classification in task-driven dialog systems.

@InProceedings{acl19disentangle,
  author    = {Jonathan K. Kummerfeld and Sai R. Gouravajhala and Joseph J. Peper and Vignesh Athreya and Chulaka Gunasekara and Jatin Ganhotra and Siva Sankalp Patel and Lazaros Polymenakos and Walter S. Lasecki},
  title     = {A Large-Scale Corpus for Conversation Disentanglement},
  location  = {Florence, Italy},
  booktitle = {Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
  month     = {July},
  year      = {2019},
  url       = {https://www.aclweb.org/anthology/P19-1374.pdf},
  pages     = {3846--3856},
  arxiv     = {https://arxiv.org/abs/1810.11118},
  software  = {https://jkk.name/irc-disentanglement},
  data      = {https://jkk.name/irc-disentanglement},
  blog_post = {/post/2019-07-10_disentanglement/},
}

      

Disentangling conversations mixed together in a single stream of messages is a difficult task, made harder by the lack of large manually annotated datasets. We created a new dataset of 77,563 messages manually annotated with reply-structure graphs that both disentangle conversations and define internal conversation structure. Our data is 16 times larger than all previously released datasets combined, the first to include adjudication of annotation disagreements, and the first to include context. We use our data to re-examine prior work, in particular, finding that 89% of conversations in a widely used dialogue corpus are either missing messages or contain extra messages. Our manually-annotated data presents an opportunity to develop robust data-driven methods for conversation disentanglement, which will help advance dialogue research.

Multi-level Context Response Matching in Retrieval-Based Dialog Systems, Basma El Amel Boussaha, Nicolas Hernandez, Christine Jacquin, Emmanuel Morin, DSTC, 2019

RAP-Net: Recurrent Attention Pooling Networks for Dialogue Response Selection, Chao-Wei Huang, Ting-Rui Chiang, Shang-Yu Su, Yun-Nung Chen, DSTC, 2019

Enhanced Sequential Representation Augmented with Utterance-level Attention for Response Selection, Taesun Whang, Dongyub Lee, Chanhee Lee, Heuiseok Lim, DSTC, 2019

Sequential Attention-based Network for Noetic End-to-End Response Selection, Qian Chen, Wen Wang, DSTC, 2019

Spatio-Temporal Matching Network for Multi-Turn Responses Selection in Retrieval-Based Chatbots, Junyu Lu, Zeying Xie, Guang Ling, Chao Zhou, Zenglin Xu, DSTC, 2019

Noetic End-to-End Response Selection with Supervised Neural Network Based Classifiers and Unsupervised Similarity Models, Pawel Skorzewski, Weronika Sieinska, Marek Kubis, DSTC, 2019

Convolutional Neural Encoder for the 7th Dialogue System Technology Challenge, Mandy Korpusik, James Glass, DSTC, 2019

Learning Multi-Level Information for Dialogue Response Selection by Highway Recurrent Transformer, Ting-Rui Chiang, Chao-Wei Huang, Shang-Yu Su, Yun-Nung Chen, DSTC, 2019

Knowledge-incorporating ESIM models for Response Selection in Retrieval-based Dialog Systems, Jatin Ganhotra, Siva Sankalp Patel, Kshitij Fadnis, DSTC, 2019

End-to-End Question Answering Models for Goal-Oriented Dialog Learning, Jamin Shin1, Andrea Madotto, Minjoon Seo, Pascale Fung, DSTC, 2019

Building Sequential Inference Models for End-to-End Response Selection, Jia-Chen Gu, Zhen-Hua Ling, Yu-Ping Ruan and Quan Liu1, DSTC, 2019

Query-Focused Scenario Construction Su Wang, Greg Durrett, Katrin Erk, EMNLP, 2019

Constructing Interpretive Spatio-Temporal Features for Multi-Turn Responses Selection, Junyu Lu, Chenbin Zhang, Zeying Xie, Guang Ling, Tom Chao Zhou, Zenglin Xu, ACL, 2019

@InProceedings{acl19slate,
  title     = {SLATE: A Super-Lightweight Annotation Tool for Experts},
  author    = {Jonathan K. Kummerfeld},
  booktitle = {Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics: System Demonstrations},
  location  = {Florence, Italy},
  month     = {July},
  year      = {2019},
  pages     = {7--12},
  url       = {https://www.aclweb.org/anthology/P19-3002.pdf},
  software  = {https://jkk.name/slate},
}

      

Many annotation tools have been developed, covering a wide variety of tasks and providing features like user management, pre-processing, and automatic labeling. However, all of these tools use a Graphical User Interface, and often require substantial effort for installation and configuration. This paper presents a new annotation tool that is designed to fill the niche of a lightweight interface for users with a terminal-based workflow. Slate supports annotation at different scales (spans of characters, tokens, and lines, or a document) and of different types (free text, labels, and links), with easily customisable keybindings, and unicode support. In a user study comparing with other tools it was consistently the easiest to install and use. Slate fills a need not met by existing systems, and has already been used to annotate two corpora, one of which involved over 250 hours of annotation effort.

@InProceedings{naacl19outliers,
  title     = {Outlier Detection for Improved Data Quality and Diversity in Dialog Systems},
  author    = {Stefan Larson and Anish Mahendran and Andrew Lee and Jonathan K. Kummerfeld and Parker Hill and Michael Laurenzano and Johann Hauswald and Lingjia Tang and Jason Mars},
  year      = {2019},
  booktitle = {Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)},
  month     = {June},
  arxiv     = {https://arxiv.org/abs/1904.03122},
  url       = {https://www.aclweb.org/anthology/N19-1051.pdf},
}

      

In a corpus of data, outliers are either errors: mistakes in the data that are counterproductive, or are unique: informative samples that improve model robustness. Identifying outliers can lead to better datasets by (1) removing noise in datasets and (2) guiding collection of additional data to fill gaps. However, the problem of detecting both outlier types has received relatively little attention in NLP, particularly for dialog systems. We introduce a simple and effective technique for detecting both erroneous and unique samples in a corpus of short texts using neural sentence embeddings combined with distance-based outlier detection. We also present a novel data collection pipeline built atop our detection technique to automatically and iteratively mine unique data samples while discarding erroneous samples. Experiments show that our outlier detection technique is effective at finding errors while our data collection pipeline yields highly diverse corpora that in turn produce more robust intent classification and slot-filling models.

@InProceedings{cicling19personal,
  title     = {Look Who's Talking: Inferring Speaker Attributes from Personal Longitudinal Dialog},
  year      = {2019},
  author    = {Charles Welch and Ver{\'o}nica P{\'e}rez-Rosas and Jonathan K. Kummerfeld and Rada Mihalcea},
  address   = {La Rochelle, France},
  booktitle = {Proceedings of the 20th International Conference on Computational Linguistics and Intelligent Text Processing (CICLing)},
  publisher = {Springer},
  month     = {April},
  awards    = {Best Student Paper Award},
  url       = {https://arxiv.org/abs/1904.11610},
  arxiv     = {https://arxiv.org/abs/1904.11610},
}

      

We examine a large dialog corpus obtained from the conversation history of a single individual with 104 conversation partners. The corpus consists of half a million instant messages, across several messaging platforms. We focus our analyses on seven speaker attributes, each of which partitions the set of speakers, namely: gender; relative age; family member; romantic partner; classmate; co-worker; and native to the same country. In addition to the content of the messages, we examine conversational aspects such as the time messages are sent, messaging frequency, psycholinguistic word categories, linguistic mirroring, and graph-based features reflecting how people in the corpus mention each other. We present two sets of experiments predicting each attribute using (1) short context windows; and (2) a larger set of messages. We find that using all features leads to gains of 9-14% over using message text only.

@InProceedings{dstc19task1,
  title     = {DSTC7 Task 1: Noetic End-to-End Response Selection},
  author    = {Chulaka Gunasekara and Jonathan K. Kummerfeld and Lazaros Polymenakos and and Walter S. Lasecki},
  year      = {2019},
  booktitle = {7th Edition of the Dialog System Technology Challenges at AAAI 2019},
  month     = {January},
  url       = {http://www.jkk.name/pub/ws18dstc_task1.pdf},
  alt-url   = {http://workshop.colips.org/dstc7/papers/dstc7_task1_final_report.pdf},
}

      

Goal-oriented dialogue in complex domains is an extremely challenging problem and there are relatively few datasets. This task provided two new resources that presented different challenges: one was focused but small, while the other was large but diverse. We also considered several new variations on the next utterance selection problem: (1) increasing the number of candidates, (2) including paraphrases, and (3) not including a correct option in the candidate set. Twenty teams participated, developing a range of neural network models, including some that successfully incorporated external data to boost performance. Both datasets have been publicly released, enabling future work to build on these results, working towards robust goal-oriented dialogue systems.

Extracting Dialog Structure and Latent Beliefs from Dialog Corpus Aishwarya Chhabra, Pratik Saini, Chandrasekhar Anantaram, LaCATODA-BtG Workshop, 2019

@Article{ieee19personal,
  author    = {Charles Welch and Ver{\'o}nica P{\'e}rez-Rosas and Jonathan K. Kummerfeld and Rada Mihalcea},
  editor    = {Erik Cambria},
  title     = {Learning from Personal Longitudinal Dialog Data},
  year      = {2019},
  journal   = {IEEE Intelligent Systems},
  volume    = {34},
  number    = {4},
  pages     = {16--23},
  url       = {https://ieeexplore.ieee.org/document/8844687},
}

      

We explore the use of longitudinal dialog data for two dialog prediction tasks: next message prediction and response time prediction. We show that a neural model using personal data that leverages a combination of message content, style matching, time features, and speaker attributes leads to the best results for both tasks, with error rate reductions of up to 15% compared to a classifier that relies exclusively on message content and to a classifier that does not use personal data.

@InProceedings{ws18dstc,
  title     = {Dialog System Technology Challenge 7},
  author    = {Koichiro Yoshino and Chiori Hori and Julien Perez and Luis Fernando D'Haro and Lazaros Polymenakos and Chulaka Gunasekara and Walter S. Lasecki and Jonathan K. Kummerfeld and Michel Galley and Chris Brockett and Jianfeng Gao and Bill Dolan and Xiang Gao and Huda Alamari and Tim K. Marks and Devi Parikh and Dhruv Batra},
  year      = {2018},
  booktitle = {NeurIPS Workshop: The 2nd Conversational AI: "Today's Practice and Tomorrow's Potential"},
  url       = {https://arxiv.org/abs/1901.03461},
  location  = {Montreal, Quebec, Canada},
  month     = {December},
  arxiv     = {https://arxiv.org/abs/1901.03461},
}

      

This paper introduces the Seventh Dialog System Technology Challenges (DSTC), which use shared datasets to explore the problem of building dialog systems. Recently, end-to-end dialog modeling approaches have been applied to various dialog tasks. The seventh DSTC (DSTC7) focuses on developing technologies related to end-to-end dialog systems for (1) sentence selection, (2) sentence generation and (3) audio visual scene aware dialog. This paper summarizes the overall setup and results of DSTC7, including detailed descriptions of the different tracks and provided datasets. We also describe overall trends in the submitted systems and the key results. Each track introduced new datasets and participants achieved impressive results using state-of-the-art end-to-end technologies.

Multi-level Context Response Matching in Retrieval-Based Dialog Systems, Basma El Amel Boussaha, Nicolas Hernandez, Christine Jacquin, Emmanuel Morin, DSTC, 2019

RAP-Net: Recurrent Attention Pooling Networks for Dialogue Response Selection, Chao-Wei Huang, Ting-Rui Chiang, Shang-Yu Su, Yun-Nung Chen, DSTC, 2019

Enhanced Sequential Representation Augmented with Utterance-level Attention for Response Selection, Taesun Whang, Dongyub Lee, Chanhee Lee, Heuiseok Lim, DSTC, 2019

Sequential Attention-based Network for Noetic End-to-End Response Selection, Qian Chen, Wen Wang, DSTC, 2019

Spatio-Temporal Matching Network for Multi-Turn Responses Selection in Retrieval-Based Chatbots, Junyu Lu, Zeying Xie, Guang Ling, Chao Zhou, Zenglin Xu, DSTC, 2019

Noetic End-to-End Response Selection with Supervised Neural Network Based Classifiers and Unsupervised Similarity Models, Pawel Skorzewski, Weronika Sieinska, Marek Kubis, DSTC, 2019

Convolutional Neural Encoder for the 7th Dialogue System Technology Challenge, Mandy Korpusik, James Glass, DSTC, 2019

Learning Multi-Level Information for Dialogue Response Selection by Highway Recurrent Transformer, Ting-Rui Chiang, Chao-Wei Huang, Shang-Yu Su, Yun-Nung Chen, DSTC, 2019

End-to-end Gated Self-attentive Memory Network for Dialog Response Selection, Shuo Sun, Yik-Cheung Tam, Jie Cao, Canxiang Yan, Zuohui Fu, Cheng Niu, Jie Zhou, DSTC, 2019

Entropy-Enhanced Multimodal Attention Model for Scene-Aware Dialogue Generation, Kuan-Yen Lin, Chao-Chun Hsu, Yun-Nung Chen, Lun-Wei Ku, DSTC, 2019

Comparison of Transfer-Learning Approaches for Response Selection in Multi-Turn Conversations, Jesse Vig, Kalai Ramea, DSTC, 2019

Top-K Attention Mechanism for Complex Dialogue System, Chang-Uk Shina, Jeong-Won Chab, DSTC, 2019

Knowledge-Grounded Response Generation with Deep Attentional Latent-Variable Model, Hao-Tong Ye, Kai-Ling Lo, Shang-Yu Su, Yun-Nung Chen, DSTC, 2019

The OneConn-MemNN System for Knowledge-Grounded Conversation Modeling, Junyuan Zheng, Surya Kasturi, Mason Lin, Xin Chen, Onkar Salvi, Harry Jiannan Wang, DSTC, 2019

An Ensemble Dialogue System for Facts-Based Sentence Generation, Ryota Tanaka, Akihide Ozeki, Shugo Kato, Akinobu Lee, DSTC, 2019

Conversing by Reading: Contentful Neural Conversation with On-demand Machine Reading, Lianhui Qin, Michel Galley, Chris Brockett, Xiaodong Liu, Xiang Gao, Bill Dolan, Yejin Choi, Jianfeng Gao, ACL, 2019

WCIS 2019: 1st Workshop on Conversational Interaction Systems Abhinav Rastogi, Alexandros Papangelis, Rahul Goel, Chandra Khatri, Proceedings of the 42Nd International ACM SIGIR Conference on Research and Development in Information Retrieval, 2019

@InProceedings{acl18sql,
  author    = {Catherine Finegan-Dollak\* and Jonathan K. Kummerfeld\* and Li Zhang and Karthik Ramanathan and Sesh Sadasivam and Rui Zhang and Dragomir Radev},
  title     = {Improving Text-to-SQL Evaluation Methodology},
  booktitle = {Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  month     = {July},
  year      = {2018},
  address   = {Melbourne, Victoria, Australia},
  pages     = {351--360},
  url       = {http://www.aclweb.org/anthology/P18-1033.pdf},
  arxiv     = {https://arxiv.org/abs/1806.09029},
  software  = {https://jkk.name/text2sql-data},
  data      = {https://jkk.name/text2sql-data},
}

      

To be informative, an evaluation must measure how well systems generalize to realistic unseen data. We identify limitations of and propose improvements to current evaluations of text-to-SQL systems. First, we compare human-generated and automatically generated questions, characterizing properties of queries necessary for real-world applications. To facilitate evaluation on multiple datasets, we release standardized and improved versions of seven existing datasets and one new text-to-SQL dataset. Second, we show that the current division of data into training and test sets measures robustness to variations in the way questions are asked, but only partially tests how well systems generalize to new queries; therefore, we propose a complementary dataset split for evaluation of future work. Finally, we demonstrate how the common practice of anonymizing variables during evaluation removes an important challenge of the task. Our observations highlight key difficulties, and our methodology enables effective measurement of future development.

Dependency-based Hybrid Trees for Semantic Parsing, Zhanming Jie, Wei Lu, EMNLP, 2018

Towards Complex Text-to-SQL in Cross-Domain Database with Intermediate Representation, Jiaqi Guo, Zecheng Zhan, Yan Gao, Yan Xiao, Jian-Guang Lou, Ting Liu, and Dongmei Zhang, ACL, 2019

Bootstrapping an End-to-End Natural Language Interface for Databases Nathaniel Weir, Prasetya Utama, SIGMOD, 2019

Neural Semantic Parsing with Anonymization for Command Understanding in General-Purpose Service Robots Nick Walker, Yu-Tang Peng, Maya Cakmak, RoboCup 2019: Robot World Cup XXIII, 2019

A cross-domain natural language interface to databases using adversarial text method Wenlu Wang, The VLDB PhD Workshop, 2019

Don't paraphrase, detect! Rapid and Effective Data Collection for Semantic Parsing, Jonathan Herzig, Jonathan Berant, EMNLP, 2019

SpatialNLI: A Spatial Domain Natural Language Interface to Databases Using Spatial Comprehension Jingjing Li, Wenlu Wang, Wei-Shinn Ku, Yingtao Tian, Haixun Wang, ACM SIGSPATIAL International Conference on Advances in Geographic Information Systems, 2019

Editing-Based SQL Query Generation for Cross-Domain Context-Dependent Questions Rui Zhang, Tao Yu, He Yang Er, Sungrok Shim, Eric Xue, Xi Victoria Lin, Tianze Shi, Caiming Xiong, Richard Socher, Dragomir Radev, EMNLP, 2019

Model-based Interactive Semantic Parsing: A Unified Formulation and A Text-to-SQL Case Study Ziyu Yao, Yu Su, Huan Sun, Wen-tau Yih, EMNLP, 2019

@InProceedings{naacl18data,
  author    = {Yiping Kang and Yunqi Zhang and Jonathan K. Kummerfeld and Parker Hill and Johann Hauswald and Michael A. Laurenzano and Lingjia Tang and Jason Mars},
  title     = {Data Collection for a Production Dialogue System: A Startup Perspective},
  booktitle = {Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 3 (Industry Papers)},
  pages     = {33--40},
  year      = {2018},
  month     = {June},
  location  = {New Orleans, Louisiana, USA},
  url       = {http://www.aclweb.org/anthology/N18-3005.pdf},
  video     = {https://vimeo.com/277631102},
}

      

Industrial dialogue systems such as Apple Siri and Google Assistant require large scale diverse training data to enable their sophisticated conversation capabilities. Crowdsourcing is a scalable and inexpensive data collection method, but collecting high quality data efficiently requires thoughtful orchestration of crowdsourcing jobs. Prior study of data collection process has focused on tasks with limited scope and performed intrinsic data analysis, which may not be indicative of impact on trained model performance. In this paper, we present a study of crowdsourcing methods for a user intent classification task in one of our deployed dialogue systems. Our task requires classification over 47 possible user intents and contains many intent pairs with subtle differences. We consider different crowdsourcing job types and job prompts, quantitatively analyzing the quality of collected data and downstream model performance on a test set of real user queries from production logs. Our observations provide insight into how design decisions impact crowdsourced data quality, with clear recommendations for future data collection for dialogue systems.

Enhancing Domain-Specific Supervised Natural Language Intent Classification with a Top-Down Selective Ensemble Model, Gard B. Jenset, Barbara McGillivray, Machine Learning and Knowledge Extraction, 2019

A Study of Incorrect Paraphrases in Crowdsourced User Utterances, Mohammad-Ali Yaghoub-Zadeh-Fard, Boualem Benatallah, Moshe Chai Barukh, Shayan Zamanirad, NAACL, 2019

Data Collection Methods for Building a Free Response Training Simulation Vaibhav Sharma, Beni Shpringer, Sung Min Yang, Martin Bolger, Sodiq Adewole, Dr. D. Brown, Erfaneh Gharavi, Systems and Information Engineering Design Symposium, 2019

Personalizing crowdsourced human-robot interaction through curiosity-driven learning Phoebe Liu, Malcolm Doering, Dylan F. Glas, Takayuki Kanda, Dana Kulic, Hiroshi Ishiguro, Personalization in Long-Term Human-Robot Interaction, 2019

@InProceedings{naacl18summary,
  author    = {Youxuan Jiang and Catherine Finegan-Dollak and Jonathan K. Kummerfeld and Walter Lasecki},
  title     = {Effective Crowdsourcing for a New Type of Summarization Task},
  booktitle = {Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 2 (Short Papers)},
  pages     = {628--633},
  year      = {2018},
  month     = {June},
  location  = {New Orleans, Louisiana, USA},
  url       = {http://www.aclweb.org/anthology/N18-2099.pdf},
}

      

Most summarization research focuses on summarizing the entire given text, but in practice readers are often interested in only one aspect of the document or conversation. We propose ‘targeted summarization’ as an umbrella category for summarization tasks that intentionally consider only parts of the input data. This covers query-based summarization, update summarization, and a new task we propose where the goal is to summarize a particular aspect of a document. However, collecting data for this new task is hard because directly asking annotators (e.g., crowd workers) to write summaries leads to data with low accuracy when there are a large number of facts to include. We introduce a novel crowdsourcing workflow, Pin-Refine, that allows us to collect highquality summaries for our task, a necessary step for the development of automatic systems.

A Crowdsourcing Approach to Evaluate the Quality of Query-based Extractive Text Summaries N. Iskender, A. Gabryszak, T. Polzehl, L. Hennig, S. Möller, Conference on Quality of Multimedia Experience, 2019

{HEIDL}: Learning Linguistic Expressions with Deep Learning and Human-in-the-Loop, Prithviraj Sen, Yunyao Li, Eser Kogan, Yiwei Yang and Walter Lasecki, ACL (demo), 2019

Efficient Elicitation Approaches to Estimate Collective Crowd Answers, John Joon Young Chung, Jean Y. Song, Sindhu Kutty, Sungsoo (Ray) Hong, Juho Kim, Walter S. Lasecki, CSCW, 2019

Efficient Elicitation Approaches to Estimate Collective Crowd Answers John Joon Young Chung, Jean Y. Song, Sindhu Kutty, Sungsoo (Ray) Hong, Juho Kim, Walter S. Lasecki, CSCW, 2019

@InProceedings{naacl18embeddings,
  author    = {Laura Wendlandt and Jonathan K. Kummerfeld and Rada Mihalcea},
  title     = {Factors Influencing the Surprising Instability of Word Embeddings},
  booktitle = {Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)},
  year      = {2018},
  month     = {June},
  location  = {New Orleans, Louisiana, USA},
  url       = {http://www.aclweb.org/anthology/N18-1190.pdf},
  pages     = {2092--2102},
  arxiv     = {https://arxiv.org/abs/1804.09692},
}

      

Despite the recent popularity of word embedding methods, there is only a small body of work exploring the limitations of these representations. In this paper, we consider one aspect of embedding spaces, namely their stability. We show that even relatively high frequency words (100-200 occurrences) are often unstable. We provide empirical evidence for how various factors contribute to the stability of word embeddings, and we analyze the effects of stability on downstream tasks.

Subcharacter information in japanese embeddings: when is it worth it? Marzena Karpinska, Bofang Li, Anna Rogers, and Aleksandr Drozd, Proceedings of the Workshop on Relevance of Linguistic Structure in Neural Architectures for NLP (RELNLP), 2018

What's in Your Embedding, And How It Predicts Task Performance Anna Rogers, Shashwath Hosur Ananthakrishna, and Anna Rumshisky, CoLing, 2018

CluWords: Exploiting Semantic Word Clustering Representation for Enhanced Topic Modeling, Felipe Viegas, Sergio D. Canuto, Christian Gomes, Washington Luiz, Thierson Rosa, Sabir Ribas, Leonardo C. da Rocha and Marcos Andre Goncalves, WSDM, 2019

Investigating the Stability of Concrete Nouns in Word Embeddings, Bénédicte Pierrejean, Ludovic Tanguy, Proceedings of the 13th International Conference on Computational Semantics - Short Papers, 2019

Modeling Word Emotion in Historical Language: Quantity Beats Supposed Stability in Seed Word Selection, Johannes Hellrich, Sven Buechel, Udo Hahn, Proceedings of the 3rd Joint SIGHUM Workshop on Computational Linguistics for Cultural Heritage, Social Sciences, Humanities and Literature, 2019

Data Shift in Legal AI Systems Venkata Nagaraju Buddarapu, Arunprasath Shankar, Workshop on Automated Semantic Analysis of Information in Legal Text, 2019

Estimating Topic Modeling Performance with Sharma–Mittal Entropy Sergei Koltcov, Vera Ignatenko and Olessia Koltsova, Entropy, 2019

A framework for anomaly detection using language modeling, and its applications to finance, Armineh Nourbakhsh, Grace Bang, 2nd KDD Workshop on Anomaly Detection in Finance, 2019

Weighted posets: Learning surface order from dependency trees William Dyer, Syntaxfest, 2019

Comparing the Performance of Feature Representations for the Categorization of the Easy-to-Read Variety vs Standard Language Marina Santini, Benjamin Danielsson, Arne Jonsson, NoDaLiDa, 2019

A Metrological Framework for Evaluating Crowd-powered Instruments, Chris Welty, Lora Aroyo, Praveen Paritosh, HComp, 2019

@InProceedings{lrec18amr,
  author    = {Charles Welch and Jonathan K. Kummerfeld and Song Feng and Rada Mihalcea},
  title     = {World Knowledge for Abstract Meaning Representation Parsing},
  booktitle = {Proceedings of the Eleventh International Conference on Language Resources and Evaluation (LREC 2018)},
  year      = {2018},
  month     = {May},
  location  = {Miyazaki, Japan},
  url       = {http://www.lrec-conf.org/proceedings/lrec2018/pdf/1085.pdf},
}

      

In this paper we explore the role played by world knowledge in semantic parsing. We look at the types of errors that currently exist in a state-of-the-art Abstract Meaning Representation (AMR) parser, and explore the problem of how to integrate world knowledge to reduce these errors. We look at three types of knowledge from (1) WordNet hypernyms and super senses, (2) Wikipedia entity links, and (3) retraining a named entity recognizer to identify concepts in AMR. The retrained entity recognizer is not perfect and cannot recognize all concepts in AMR and we examine the limitations of the named entity features using a set of oracles. The oracles show how performance increases if it can recognize different subsets of AMR concepts. These results show improvement on multiple fine-grained metrics, including a 6% increase in named entity F-score, and provide insight into the potential of world knowledge for future work in Abstract Meaning Representation parsing.

Towards Turkish Abstract Meaning Representation, Zahra Azin, Gulcsen Eryigit, ACL: SRW, 2019

@InProceedings{emnlp17forums,
  author    = {Greg Durrett and Jonathan K. Kummerfeld and Taylor Berg-Kirkpatrick and Rebecca S. Portnoff and Sadia Afroz and Damon McCoy and Kirill Levchenko  and Vern Paxson},
  title     = {Identifying Products in Online Cybercrime Marketplaces: A Dataset for Fine-grained Domain Adaptation},
  arxiv     = {https://arxiv.org/abs/1708.09609},
  booktitle = {Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
  year      = {2017},
  month     = {September},
  location  = {Copenhagen, Denmark},
  url       = {https://www.aclweb.org/anthology/D17-1275.pdf},
  pages     = {2588--2597},
  software  = {https://evidencebasedsecurity.org/forums/},
}

      

One weakness of machine-learned NLP models is that they typically perform poorly on out-of-domain data. In this work, we study the task of identifying products being bought and sold in online cybercrime forums, which exhibits particularly challenging cross-domain effects. We formulate a task that represents a hybrid of slot-filling information extraction and named entity recognition and annotate data from four different forums. Each of these forums constitutes its own ‘fine-grained domain’ in that the forums cover different market sectors with different properties, even though all forums are in the broad domain of cybercrime. We characterize these domain differences in the context of a learning-based system: supervised models see decreased accuracy when applied to new forums, and standard techniques for semi-supervised learning and domain adaptation have limited effectiveness on this data, which suggests the need to improve these techniques. We release a dataset of 1,938 annotated posts from across the four forums.

You Are Your Photographs: Detecting Multiple Identities of Vendors in the Darknet Marketplaces Xiangwen Wang, Peng Peng, Chun Wang, and Gang Wang, ASIA CCS, 2018

Reading Thieves' Cant: Automatically Identifying and Understanding Dark Jargons from Cybercrime Marketplaces, Kan Yuan, Haoran Lu, Xiaojing Liao, and XiaoFeng Wang, USENIX, 2018

Automatically identifying the function and intent of posts in underground forums, Andrew Caines, Sergio Pastrana, Alice Hutchings, and Paula J. Buttery, Crime Science, 2018

Understanding and Detecting Private Interactions in Underground Forums Zhibo Sun, Carlos E. Rubio-Medrano, Ziming Zhao, Tiffany Bao, Adam Doupe and Gail-Joon Ahn, ACM Conference on Data and Application Security and Privacy (CODASPY), 2019

Understanding and Predicting Private Interactions in Underground Forums Zhibo Sun, Carlos E. Rubio-Medrano, Ziming Zhao, Tiffany Bao, Adam Doupe and Gail-Joon Ahn, Proceedings of the Ninth ACM Conference on Data and Application Security and Privacy, 2019

@InProceedings{acl17paraphrase,
  author    = {Youxuan Jiang and Jonathan K. Kummerfeld and Walter S. Lasecki},
  title     = {Understanding Task Design Trade-offs in Crowdsourced Paraphrase Collection},
  booktitle = {Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
  month     = {July},
  year      = {2017},
  location  = {Vancouver, Canada},
  pages     = {103--109},
  url       = {http://www.aclweb.org/anthology/P17-2017.pdf},
  data      = {http://aclweb.org/anthology/attachments/P/P17/P17-2017.Datasets.zip},
  video     = {https://vimeo.com/234958413},
  arxiv     = {https://arxiv.org/abs/1704.05753},
}

      

Linguistically diverse datasets are critical for training and evaluating robust machine learning systems, but data collection is a costly process that often requires experts. Crowdsourcing the process of paraphrase generation is an effective means of expanding natural language datasets, but there has been limited analysis of the trade-offs that arise when designing tasks. In this paper, we present the first systematic study of the key factors in crowdsourcing paraphrase collection. We consider variations in instructions, incentives, data domains, and workflows. We manually analyzed paraphrases for correctness, grammaticality, and linguistic diversity. Our observations provide new insight into the trade-offs between accuracy and diversity in crowd responses that arise as a result of task design, providing guidance for future paraphrase generation procedures.

Effective Crowdsourced Generation of Training Data for Chatbots Natural Language Understanding, R. Bapat, P. Kucherbaev, and A. Bozzon, ICWE, 2018

SPADE: Evaluation Dataset for Monolingual Phrase Alignment, Yuki Arase1, Junichi Tsujii?, LREC, 2018

Crowdsourcing for Reminiscence Chatbot Design, Svetlana Nikitina, Florian Daniel, Marcos Baez, Fabio Casati, HCOMP, 2018

Towards More Robust Speech Interactions for Deaf and Hard of Hearing Users, Raymond Fok, Harmanpreet Kaur, Skanda Palani, Martez E. Mott and Walter S. Lasecki, ASSETS, 2018

A Study of Incorrect Paraphrases in Crowdsourced User Utterances, Mohammad-Ali Yaghoub-Zadeh-Fard, Boualem Benatallah, Moshe Chai Barukh, Shayan Zamanirad, NAACL, 2019

Optimizing the Design and Cost for Crowdsourced Conversational Utterances Phoebe Liu, Joan Xiao, Tong Liu, Dylan F. Glas, Workshop on Data Collection, Curation, and Labeling (DCCL) for Mining and Learning, 2019

Personalizing crowdsourced human-robot interaction through curiosity-driven learning Phoebe Liu, Malcolm Doering, Dylan F. Glas, Takayuki Kanda, Dana Kulic, Hiroshi Ishiguro, Personalization in Long-Term Human-Robot Interaction, 2019

PKU Paraphrase Bank: A Sentence-Level Paraphrase Corpus for Chinese Bowei Zhang, Weiwei Sun, Xiaojun Wan, Zongming Guo, Natural Language Processing and Chinese Computing, 2019

Efficient Elicitation Approaches to Estimate Collective Crowd Answers John Joon Young Chung, Jean Y. Song, Sindhu Kutty, Sungsoo (Ray) Hong, Juho Kim, Walter S. Lasecki, CSCW, 2019

@InProceedings{www17forums,
  title     = {Tools for Automated Analysis of Cybercriminal Markets},
  author    = {Rebecca S. Portnoff and Sadia Afroz and Greg Durrett and Jonathan K. Kummerfeld and Taylor Berg-Kirkpatrick and Damon McCoy and Kirill Levchenko and Vern Paxson},
  booktitle = {Proceedings of 26th International World Wide Web conference},
  month     = {April},
  year      = {2017},
  location  = {Perth, Australia},
  url       = {http://www.jkk.name/pub/www17forums.pdf},
  software  = {http://evidencebasedsecurity.org/forums/},
}

      

Underground forums are widely used by criminals to buy and sell a host of stolen items, datasets, resources, and criminal services. These forums contain important resources for understanding cybercrime. However, the number of forums, their size, and the domain expertise required to understand the markets makes manual exploration of these forums unscalable. In this work, we propose an automated, top-down approach for analyzing underground forums. Our approach uses natural language processing and machine learning to automatically generate high-level information about underground forums, first identifying posts related to transactions, and then extracting products and prices. We also demonstrate, via a pair of case studies, how an analyst can use these automated approaches to investigate other categories of products and transactions. We use eight distinct forums to assess our tools: Antichat, Blackhat World, Carders, Darkode, Hack Forums, Hell, L33tCrew and Nulled. Our automated approach is fast and accurate, achieving over 80% accuracy in detecting post category, product, and prices.

Ethical issues of research using datasets of illicit origin, Daniel R. Thomas, Sergio Pastrana, Alice Hutchings, Richard Clayton, and Alastair R. Beresford, IMC, 2017

CrimeBB: Enabling Cybercrime Research on Underground Forums at Scale, Sergio Pastrana, Daniel R. Thomas, Alice Hutchings, and Richard Clayton, WWW, 2018

At-risk system identification via analysis of discussions on the darkweb, Eric Nunes, Paulo Shakarian, and Gerardo I. Simari, APWG Symposium on Electronic Crime Research (eCrime), 2018

Systematically Understanding the Cyber Attack Business: A Survey, Keman Huang, Michael Siegel, and Stuart Madnick, ACM Computing Surveys, 2018

Characterizing Eve: Analysing Cybercrime Actors in a Large Underground Forum Sergio Pastrana, Alice Hutchings, Andrew Caines, and Paula Buttery, International Symposium on Research in Attacks, Intrusions and Defenses, 2018

iDetector: Automate Underground Forum Analysis Based on Heterogeneous Information Network Yiming Zhang, Yujie Fan, Shifu Hou, Jian Liu, Yanfang Ye, and Thirimachos Bourlai, IEEE/ACM International Conference on Advances in Social Networks Analysis and Mining (ASONAM), 2018

Automatically identifying the function and intent of posts in underground forums, Andrew Caines, Sergio Pastrana, Alice Hutchings, and Paula J. Buttery, Crime Science, 2018

Analyzing and Identifying Data Breaches in Underground Forums, Yong Fang, Yusong Guo, Cheng Huang, and Liang Liu, IEEE Access, 2019

Multistream Classification for Cyber Threat Data with Heterogeneous Feature Space, Yi-Fan Li, Yang Gao, Gbadebo Ayoade, Hemeng Tao, Latifur Khan, Bhavani Thuraisingham, WWW, 2019

An Empirical Study of Malicious Threads in Security Forums, Joobin Gharibshah, Zhabiz Gharibshah, Evangelos E. Papalexakis, and Michalis Faloutsos, WWW, 2019

CARONTE: Crawling Adversarial Resources Over Non-Trusted, High-Profile Environments M. Campobasso, P. Burda, L. Allodi, 2019 IEEE European Symposium on Security and Privacy Workshops, 2019

Chapter 3 - Challenges of using machine learning algorithms for cybersecurity: a study of threat-classification models applied to social media communication data Andrei Queiroz Lima, Brian Keegan, , 2020

The Art and Craft of Fraudulent App Promotion in Google Play Mizanur Rahman, Nestor Hernandez, Ruben Recabarren, Syed Ishtiaque Ahmed, Bogdan Carbunar, CCS, 2019

@InProceedings{tacl17parsing,
  author    = {Jonathan K. Kummerfeld and Dan Klein},
  title     = {Parsing with Traces: An O($n^4$) Algorithm and a Structural Representation},
  booktitle = {Transactions of the Association for Computational Linguistics},
  volume    = {5},
  issn      = {2307-387X},
  pages     = {441--454},
  year      = {2017},
  url       = {https://www.mitpressjournals.org/doi/abs/10.1162/tacl_a_00072},
  arxiv     = {https://arxiv.org/abs/1707.04221},
  software  = {https://jkk.name/1ec-graph-parser/},
  interview = {https://soundcloud.com/nlp-highlights/46-parsing-with-traces-with-jonathan-kummerfeld},
  video     = {https://vimeo.com/238235203},
}

      

General treebank analyses are graph structured, but parsers are typically restricted to tree structures for efficiency and modeling reasons. We propose a new representation and algorithm for a class of graph structures that is flexible enough to cover almost all treebank structures, while still admitting efficient learning and inference. In particular, we consider directed, acyclic, one-endpoint-crossing graph structures, which cover most long-distance dislocation, shared argumentation, and similar tree-violating linguistic phenomena. We describe how to convert phrase structure parses, including traces, to our new representation in a reversible manner. Our dynamic program uniquely decomposes structures, is sound and complete, and covers 97.3% of the Penn English Treebank. We also implement a proofof-concept parser that recovers a range of null elements and trace types.

Exploiting Structure in Parsing to 1-Endpoint-Crossing Graphs, Robin Kurtz, Marco Kuhlmann, IWPT, 2017

An Analysis of Plane Task Text Ellipticity and the Possibility of Ellipses Reconstructing Based on Cognitive Modeling Geometric Objects and Actions, Xenia Naidenova, Sergei Kurbatov, Vjacheslav Ganapol'skii, Proceedings of Computational Models in Language and Speech Workshop, 2018

AGRR-2019: A Corpus for Gapping Resolution in Russian, Maria Ponomareva, Kira Droganova, Ivan Smurov, Tatiana Shavrina, The 7th Workshop on Balto-Slavic Natural Language Processing, 2019

@PhDThesis{thesis16parsing,
  title     = {Algorithms for Identifying Syntactic Errors and Parsing with Graph Structured Output},
  author    = {Jonathan K. Kummerfeld},
  year      = {2016},
  month     = {Aug},
  location  = {Berkeley, CA, USA},
  school    = {EECS Department, University of California, Berkeley},
  number    = {UCB/EECS-2016-138},
  url       = {https://www2.eecs.berkeley.edu/Pubs/TechRpts/2016/EECS-2016-138.html},

  In this dissertation, we present new algorithms for error analysis and parsing. The heart of our approach to error analysis is the use of structural transformations to identify more meaningful classes of errors, and to enable comparisons across formalisms. For parsing, we combine a novel dynamic program with careful choices in syntactic representation to create an efficient parser that produces graph structured output. Together, these developments allowed us to evaluate the outstanding challenges in parsing and to address a key weakness in current work.

  First, we present a search algorithm that, given two structures, finds a sequence of modifications leading from one structure to the other. We applied this algorithm to syntactic error analysis, where one structure is the output of a parser, the other is the correct parse, and each modification corresponds to fixing one error. We constructed a tool based on the algorithm and analyzed variations in behavior between parsers, types of text, and languages. Our observations shine light on several assumptions about syntactic errors, showing some to be true and others to be false. For example, prepositional phrase attachment errors are indeed a major issue, while coordination scope errors do not hurt performance as much as expected.

  Next, we describe an algorithm that builds a parse in one syntactic representation to match a parse in another representation. Specifically, we build phrase structure parses from Combinatory Categorial Grammar derivations. Our approach follows the philosophy of CCG, defining specific phrase structures for each lexical category and generic rules for combinatory steps. The new parse is built by following the CCG derivation bottom-up, gradually building the corresponding phrase structure parse. This produced significantly more accurate parses than past work, and enabled us to compare performance of several parsers across formalisms.

  Finally, we address a weakness we observed in phrase structure parsers: the exclusion of syntactic trace structures for computational convenience. We present an efficient dynamic programming algorithm that constructs the graph structure that has the highest score under an edge-factored scoring function. We define a parse representation compatible with the algorithm, and show how certain linguistic distinctions dramatically impact coverage. We also show various ways to modify the algorithm to improve performance by exploiting properties of observed linguistic structure. This approach to syntactic parsing is the first to cover virtually all structure encoded in the Penn Treebank.},
}

      

Representation of syntactic structure is a core area of research in Computational Linguistics, disambiguating distinctions in meaning that are crucial for correct interpretation of language. Development of algorithms and statistical models over the past three decades has led to systems that are accurate enough to be deployed in industry, playing a key role in products such as Google Search and Apple Siri. However, syntactic parsers today are usually constrained to tree representations of language, and performance is interpreted through a single metric that conveys no linguistic information regarding remaining errors.

In this dissertation, we present new algorithms for error analysis and parsing. The heart of our approach to error analysis is the use of structural transformations to identify more meaningful classes of errors, and to enable comparisons across formalisms. For parsing, we combine a novel dynamic program with careful choices in syntactic representation to create an efficient parser that produces graph structured output. Together, these developments allowed us to evaluate the outstanding challenges in parsing and to address a key weakness in current work.

First, we present a search algorithm that, given two structures, finds a sequence of modifications leading from one structure to the other. We applied this algorithm to syntactic error analysis, where one structure is the output of a parser, the other is the correct parse, and each modification corresponds to fixing one error. We constructed a tool based on the algorithm and analyzed variations in behavior between parsers, types of text, and languages. Our observations shine light on several assumptions about syntactic errors, showing some to be true and others to be false. For example, prepositional phrase attachment errors are indeed a major issue, while coordination scope errors do not hurt performance as much as expected.

Next, we describe an algorithm that builds a parse in one syntactic representation to match a parse in another representation. Specifically, we build phrase structure parses from Combinatory Categorial Grammar derivations. Our approach follows the philosophy of CCG, defining specific phrase structures for each lexical category and generic rules for combinatory steps. The new parse is built by following the CCG derivation bottom-up, gradually building the corresponding phrase structure parse. This produced significantly more accurate parses than past work, and enabled us to compare performance of several parsers across formalisms.

Finally, we address a weakness we observed in phrase structure parsers: the exclusion of syntactic trace structures for computational convenience. We present an efficient dynamic programming algorithm that constructs the graph structure that has the highest score under an edge-factored scoring function. We define a parse representation compatible with the algorithm, and show how certain linguistic distinctions dramatically impact coverage. We also show various ways to modify the algorithm to improve performance by exploiting properties of observed linguistic structure. This approach to syntactic parsing is the first to cover virtually all structure encoded in the Penn Treebank.

@InProceedings{emnlp15learn,
  title     = {An Empirical Analysis of Optimization for Max-Margin NLP},
  author    = {Jonathan K. Kummerfeld and Taylor Berg-Kirkpatrick and Dan Klein},
  booktitle = {Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing},
  month     = {September},
  year      = {2015},
  location  = {Lisbon, Portugal},
  pages     = {273--279},
  url       = {http://www.jkk.name/pub/emnlp15learn.pdf},
  poster    = {http://www.jkk.name/pub/emnlp15learn_poster.png},
  software  = {https://github.com/tberg12/murphy},
}

      

Despite the convexity of structured max-margin objectives (Taskar et al., 2004; Tsochantaridis et al., 2004), the many ways to optimize them are not equally effective in practice. We compare a range of online optimization methods over a variety of structured NLP tasks (coreference, summarization, parsing, etc) and find several broad trends. First, margin methods do tend to outperform both likelihood and the perceptron. Second, for max-margin objectives, primal optimization methods are often more robust and progress faster than dual methods. This advantage is most pronounced for tasks with dense or continuous-valued features. Overall, we argue for a particularly simple online primal subgradient descent method that, despite being rarely mentioned in the literature, is surprisingly effective in relation to its alternatives.

Using accelerometers to remotely and automatically characterize behavior in small animals Talisin T. Hammond, Dwight Springthorpe, Rachel E. Walsh, Taylor Berg-Kirkpatrick, Journal of Experimental Biology, 2016

Joint Models for Extracting Adverse Drug Events from Biomedical Text Fei Li, Yue Zhang, Meishan Zhang, Donghong Ji, IJCAI, 2016

A Practical Perspective on Latent Structured Prediction for Coreference Resolution, Iryna Haponchyk, Alessandro Moschitti, EACL, 2017

Fine-Grained Entity Typing with High-Multiplicity Assignments, Maxim Rabinovich, Dan Klein, ACL, 2017

Learning-Based Single-Document Summarization with Compression and Anaphoricity Constraints, Greg Durrett, Taylor Berg-Kirkpatrick, Dan Klein, ACL, 2016

Multi-Task Structured Prediction for Entity Analysis: Search-Based Learning Algorithms, Chao Ma, Janardhan Rao Doppa, Prasad Tadepalli, Hamed Shahbazi, Xiaoli Fern, ACML, 2017

Post-Specialisation: Retrofitting Vectors of Words Unseen in Lexical Resources, Ivan Vulić, Goran Glavaš, Nikola Mrkšić, and Anna Korhonen, NAACL, 2018

DGeoSegmenter: A dictionary-based Chinese word segmenter for the geoscience domain, Qinjun Qiu, Zhong Xie, Liang Wu and Wenjia Li, Computers and Geosciences, 2018

@InProceedings{emnlp13analysis,
  title     = {Error-Driven Analysis of Challenges in Coreference Resolution},
  author    = {Jonathan K. Kummerfeld and Dan Klein},
  booktitle = {Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing},
  month     = {October},
  year      = {2013},
  location  = {Seattle, Washington, USA},
  pages     = {265--277},
  software  = {https://jkk.name/berkeley-coreference-analyser/},
  url       = {http://www.aclweb.org/anthology/D13-1027.pdf},
  slides    = {http://www.jkk.name/pub/emnlp13analysis_keynote.key},
  slidespdf = {http://www.jkk.name/pub/emnlp13analysis_slides.pdf},
}

      

Coreference resolution metrics quantify errors but do not analyze them. Here, we consider an automated method of categorizing errors in the output of a coreference system into intuitive underlying error types. Using this tool, we first compare the error distributions across a large set of systems, then analyze common errors across the top ten systems, empirically characterizing the major unsolved challenges of the coreference resolution task.

Examining the Impact of Coreference Resolution on Quote Attribution Tim O'Keefe, Kellie Webster, James R. Curran, Irena Koprinska, ALTA, 2013

Visualization, Search, and Error Analysis for Coreference Annotations Markus Gartner, Anders Bjorkelund, Gregor Thiele, Wolfgang Seeker, Jonas Kuhn, ACL, 2014

Low memory incremental coreference resolution Kellie Webster, James Curran, CoLing, 2014

Linking people in videos with "their" names using coreference resolution Vignesh Ramanathan, Armand Joulin, Percy Liang, Li Fei-Fei, ECCV, 2014

Solving Hard Coreference Problems Haoruo Peng, Daniel Khashabi, Dan Roth, NAACL, 2015

Analyzing and Visualizing Coreference Resolution Errors Sebastian Martschat, Thierry G"{o}ckel, Michael Strube, NAACL, 2015

Modeling the Lifespan of Discourse Entities with Application to Coreference Resolution Marie-Catherine de Marneffe, Marta Recasens, Christopher Potts, JAIR, 2015

Distributional Semantics for Resolving Bridging Mentions Tim Feuerbach, Martin Riedl, Chris Biemann, RANLP, 2015

Using Lexical and Encyclopedic Knowledge Yannick Versley, Massimo Poesio, Simone Ponzetto, , 2015

Error analysis for anaphora resolution in Russian: new challenging issues for anaphora resolution task in a morphologically rich language Svetlana Toldova, Ilya Azerkovich, Anna Roytberg, Alina Ladygina, Maria Vasilyeva, Workshop on Coreference Resolution Beyond OntoNotes (CORBON 2016), 2016

Visual Development & Analysis of Coreference Resolution Systems with CORVIDAE Nico Moller, Gunther Heidemann, Visualization and Interaction for Ontologies and Linked Data, 2016

CORVIDAE: Coreference Resolution Visual Development & Analysis Environment Nico Moller, Gunther Heidemann, International Conference on Semantic Systems, 2016

Richer Event Description: Integrating event coreference with temporal, causal and bridging annotation Tim O'Gorman, Kristin Wright-Bettner, Martha Palmer, Proceedings of the 2nd Workshop on Computing News Storylines (CNS 2016), 2016

Multilingual coreference resolution Sandra Kubler, Desislava Zhekova, Language and Linguistics Compass, 2016

A scaffolding approach to coreference resolution integrating statistical and rule-based models Heeyoung Lee, Mihai Surdeanu, Dan Jurafsky, NLE, 2017

A method for in-depth comparative evaluation: How (dis)similar are outputs of pos taggers, dependency parsers and coreference resolvers really?, Don Tuggener, EACL, 2017

An Active Learning Approach to Coreference Resolution, Mrinmaya Sachan, Eduard Hovy, Eric P. Xing, IJCAI, 2015

Incorporating Structural Information for Better Coreference Resolution, Kong Fang, Fu Jian, IJCAI, 2019

@InProceedings{acl13analysis,
  title     = {An Empirical Examination of Challenges in Chinese Parsing},
  author    = {Jonathan K. Kummerfeld and Daniel Tse and James R. Curran and Dan Klein},
  booktitle = {Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
  location  = {Sofia, Bulgaria},
  pages     = {98--103},
  month     = {August},
  year      = {2013},
  software  = {https://jkk.name/berkeley-parser-analyser/},
  url       = {http://www.aclweb.org/anthology/P13-2018.pdf},
  slides    = {http://www.jkk.name/pub/acl13analysis_keynote.key},
  slidespdf = {http://www.jkk.name/pub/acl13analysis_slides.pdf},
}

      

Aspects of Chinese syntax result in a distinctive mix of parsing challenges. However, the contribution of individual sources of error to overall difficulty is not well understood. We conduct a comprehensive automatic analysis of error types made by Chinese parsers, covering a broad range of error types for large sets of sentences, enabling the first empirical ranking of Chinese error types by their performance impact. We also investigate which error types are resolved by using gold part-of-speech tags, showing that improving Chinese tagging only addresses certain error types, leaving substantial outstanding challenges.

Chinese syntactic parsing based on linguistic entity-relationship model, Dechun Yin, Proceedings of the 22nd ACM International Conference on Information and Knowledge Management, 2013

A Hebrew verb–complement dictionary, Hanna Fadida, Alon Itai, Shuly Wintner, LREC, 2013

Two Knives Cut Better Than One: Chinese Word Segmentation with Dual Decomposition, Mengqiu Wang, Rob Voigt, Christopher D. Manning, ACL, 2014

Joint POS Tagging and Transition-based Constituent Parsing in Chinese with Non-local Features, Zhiguo Wang, Nianwen Xue, ACL, 2014

Improved Parsing with Taxonomy of Conjunctions, Dongchen Li, Xiantao Zhang, Xihong Wu, IEEE China Summit and International Conference on Signal and Information Processing, 2014

Parsing Chinese with a Generalized Categorial Grammar, Manjuan Duan, William Schuler, Proceedings of the Grammar Engineering Across Frameworks (GEAF) 2015 Workshop, 2015

Research on Chinese Parsing Based on the Improved Compositional Vector Grammar, Jingyi Li, Lingling Mu, Hongying Zan, Kunli Zhang, 16th Workshop, CLSW 2015, Revised Selected Papers, 2015

Does String-Based Neural MT Learn Source Syntax?, Xing Shi, Unkit Padhi, Kevin Knight, EMNLP, 2016

A Semantic-Oriented Grammar for Chinese Treebanking, Meishan Zhang, Yue Zhang, Wanxiang Che, Ting Liu, CICLing, 2016

Resolving Coordinate Structures for Chinese Constituent Parsing, Yichu Zhou, Shujian Huang, Xinyu Dai, Jiajun Chen, Natural Language Processing and Chinese Computing, 2015

Non-Deterministic Segmentation for Chinese Lattice Parsing, Hai Hu, Daniel Dakota, Sandra Kubler, Proceedings of the International Conference Recent Advances in Natural Language Processing, RANLP 2017, 2017

Test Sets for Chinese Nonlocal Dependency Parsing Manjuan Duan, William Schuler, LREC, 2018

@Article{astro13clouds,
  title     = {High-velocity Clouds in the Galactic All Sky Survey. I. Catalog},
  author    = {Vanessa A. Moss and Naomi M. McClure-Griffiths and Tara Murphy and D. J. Pisano and Jonathan K. Kummerfeld and James R. Curran},
  volume    = {209},
  number    = {1},
  pages     = {12},
  publisher = {IOP Publishing},
  journal   = {The Astrophysical Journal Supplement Series},
  year      = {2013},
  url       = {http://iopscience.iop.org/0067-0049/209/1/12},
  arxiv     = {https://arxiv.org/abs/1309.4104},
}

      

We present a catalogue of high-velocity clouds (HVCs) from the Galactic All Sky Survey (GASS) of southern-sky neutral hydrogen, which has 57 mK sensitivity and 1 km/s velocity resolution and was obtained with the Parkes Telescope. Our catalogue has been derived from the stray-radiation corrected second release of GASS. We describe the data and our method of identifying HVCs and analyse the overall properties of the GASS population. We catalogue a total of 1693 HVCs at declinations < 0 deg, including 1111 positive velocity HVCs and 582 negative velocity HVCs. Our catalogue also includes 295 anomalous velocity clouds (AVCs). The cloud line-widths of our HVC population have a median FWHM of ~19 km/s, which is lower than found in previous surveys. The completeness of our catalogue is above 95% based on comparison with the HIPASS catalogue of HVCs, upon which we improve with an order of magnitude in spectral resolution. We find 758 new HVCs and AVCs with no HIPASS counterpart. The GASS catalogue will shed an unprecedented light on the distribution and kinematic structure of southern-sky HVCs, as well as delve further into the cloud populations that make up the anomalous velocity gas of the Milky Way.

HI4PI: a full-sky H i survey based on EBHIS and GASS, N. Ben Bekhti, L. Flöer, R. Keller, J. Kerp, D. Lenz and B. Winkel, J. Bailin, M. R. Calabretta, L. Dedes, H. A. Ford, B. K. Gibson, U. Haud, S. Janowiecki, P. M. W. Kalberla, F. J. Lockman, N. M. McClure-Griffiths, T. Murphy, H. Nakanishi, D. J. Pisano and L. Staveley-Smith, Astronomy and Astrophysics, 2016

Theoretical model of hydrodynamic jet formation from accretion disks with turbulent viscosity E. Arshilava, M. Gogilashvili, V. Loladze, I. Jokhadze, B. Modrekiladze, N.L. Shatashvili, A.G. Tevzadze, Journal of High Energy Astrophysics, 2019

@InProceedings{emnlp12analysis,
  title     = {Parser Showdown at the Wall Street Corral: An Empirical Investigation of Error Types in Parser Output},
  author    = {Jonathan K. Kummerfeld and David Hall and James R. Curran and Dan Klein},
  booktitle = {Proceedings of the 2012 Joint Conference on Empirical Methods in Natural Language Processing and Computational Natural Language Learning},
  year      = {2012},
  pages     = {1048--1059},
  month     = {July},
  location  = {Jeju Island, South Korea},
  software  = {https://jkk.name/berkeley-parser-analyser/},
  url       = {http://www.aclweb.org/anthology/D12-1096.pdf},
  slides    = {http://www.jkk.name/pub/emnlp12analysis_keynote.key},
  slidespdf = {http://www.jkk.name/pub/emnlp12analysis_slides.pdf},
}

      

Constituency parser performance is primarily interpreted through a single metric, F-score on WSJ section 23, that conveys no linguistic information regarding the remaining errors. We classify errors within a set of linguistically meaningful types using tree transformations that repair groups of errors together. We use this analysis to answer a range of questions about parser behaviour, including what linguistic constructions are difficult for state-of-the-art parsers, what types of errors are being resolved by rerankers, and what types are introduced when parsing out-of-domain text.

Joint Apposition Extraction with Syntactic and Semantic Constraints, Will Radford, James R. Curran, ACL, 2013

Parsing with Compositional Vector Grammars, Richard Socher, John Bauer, Christopher D. Manning, Andrew Y. Ng, ACL, 2013

A Hebrew verb-complement dictionary, Hanna Fadida, Alon Itai, Shuly Wintner, LREC, 2013

On the Elements of an Accurate Tree-to-String Machine Translation System, Graham Neubig, Kevin Duh, ACL, 2014

Parser Evaluation Using Derivation Trees: A Complement to evalb, Seth Kulick, Ann Bies, Justin Mott, Anthony Kroch, Mark Liberman, Beatrice Santorini, ACL, 2014

Joint RNN-Based Greedy Parsing and Word Composition, Joel Legrand, Ronan Collobert, ICLR, 2015

Exploring Compositional Architectures and Word Vector Representations for Prepositional Phrase Attachment, Yonatan Belinkov, Tao Lei, Regina Barzilay, Amir Globerson, TACL, 2014

Identifying Cascading Errors using Constraints in Dependency Parsing, Dominick Ng, James R. Curran, ACL, 2015

It Depends: Dependency Parser Comparison Using A Web-based Evaluation Tool, Jinho D. Choi, Joel Tetreault, Amanda Stent, ACL, 2015

Transition-based Neural Constituent Parsing, Taro Watanabe, Eiichiro Sumita, ACL, 2015

What is hard in Universal Dependency Parsing?, Angelika Kirilina, Yannick Versleya, SPMRL, 2015

A Protocol for Annotating Parser Differences, James V. Bruno, Aoife Cahill, Binod Gyawali, ETS Research Reports, 2016

Predicting the Performance of Parsing with Referential Translation Machines, Ergun Bicici, The Prague Bulletin of Mathematical Linguistics, 2016

An Evaluation of Parser Robustness for Ungrammatical Sentences, Home B. Hashemi, Rebecca Hwa, EMNLP, 2016

Fine-grained parallelism in probabilistic parsing with Habanero Java, Matthew Francis-Landau, Bing Xue, Jason Eisner, Vivek Sarkar, Workshop on Irregular Applications: Architectures and Algorithms, 2016

Old School vs. New School: Comparing Transition-Based Parsers with and without Neural Network Enhancement, Miryam de Lhoneux, Sara Stymne, Joakim Nivre, International Workshop on Treebanks and Linguistic Theories, 2017

PP Attachment: Where do We Stand?, Dani"{e}l de Kok, Jianqiang Ma, Corina Dima, Erhard Hinrichs, EACL, 2017

Deep Semantic Role Labeling: What Works and What's Next, Luheng He, Kenton Lee, Mike Lewis, Luke Zettlemoyer, ACL, 2017

Breaking {NLP}: {Using} {Morphosyntax}, {Semantics}, {Pragmatics} and {World} {Knowledge} to {Fool} {Sentiment} {Analysis} {Systems}, Taylor Mahler, Willy Cheung, Micha Elsner, David King, Marie-Catherine de Marneffe, Cory Shain, Symon Stevens-Guille, Michael White, {EMNLP} 2017 {Workshop} on {Building} {Linguistically} {Generalizable} {NLP} {Systems}, 2017

A Simple Method for Clarifying Sentences with Coordination Ambiguities, Michael White, Manjuan Duan, David L. King, INLG, 2017

Does String-Based Neural MT Learn Source Syntax?, Xing Shi, Unkit Padhi, Kevin Knight, EMNLP, 2016

Prepositional Phrase Attachment over Word Embedding Products, Pranava Swaroop Madhyastha, Xavier Carreras, Ariadna Quattoni, IWPT, 2017

Improving Sequence-to-Sequence Constituency Parsing Lemao Liu, Muhua Zhu, Shuming Shi, AAAI, 2018

Extending a Parser to Distant Domains Using a Few Dozen Partially Annotated Examples Vidur Joshi, Matthew Peters, Mark Hopkins, ACL, 2018

Madly Ambiguous: A Game for Learning about Structural Ambiguity and Why It's Hard for Computers, Ajda Gokcen, Ethan Hill, Michael White, NAACL (Demonstration), 2018

Parsing Speech: A Neural Approach to Integrating Lexical and Acoustic-Prosodic Information, Trang Tran, Shubham Toshniwal, Mohit Bansal, Kevin Gimpel, Karen Livescu and Mari Ostendorf, NAACL, 2018

Automated Extraction of Semantic Legal Metadata Using Natural Language Processing, Amin Sleimi, Nicolas Sannier, Mehrdad Sabetzadeh, Lionel Briand and John Dann, The 26th IEEE International Requirements Engineering Conference, 2018

An Empirical Investigation of Error Types in Vietnamese Parsing, Quy T. Nguyen, Yusuke Miyao, Hiroshi Noji, and Nhung T.H. Nguyen, CoLing, 2018

Sprucing up the trees - Error detection in treebanks, Ines Rehbein, Josef Ruppenhofer, CoLing, 2018

Natural Language Parsing: Progress and Challenges, Carlos Gomez-Rodriguez, Boletin de Estadistica e Investigacion Operativa, 2018

Constituent Parsing as Sequence Labeling, Carlos Gomez-Rodriguez, David Vilares, EMNLP, 2018

Natural Language Parsing: Progress and Challenges Carlos Gomez-Rodriguez, Boletin de Estadistica e Investigacion Operativa, 2018

The status of function words in dependency grammar: A critique of Universal Dependencies (UD), Timothy Osborne, Kim Gerdes, Glossa, 2019

Visual Disambiguation of Prepositional Phrase Attachments: Multimodal Machine Learning for Syntactic Analysis Correction {Sebastien Delecraz, Leonor Becerra-Bonache, Alexis Nasr, Frederic Bechet, Benoit Favre, Advances in Computational Intelligence, 2019

On the Role of Style in Parsing Speech with Neural Models Trang Tran, Jiahong Yuan, Yang Liu, Mari Ostendorf, Interspeech, 2019

Integration of a Multilingual Preordering Component into a Commercial SMT Platform Anita Ramm, Riccardo Superbo, Dimitar Shterionov, Tony O'Dowd, Alexander Fraser, The Prague Bulletin of Mathematical Linguistics, 2017

@InProceedings{acl12conversion,
  title     = {Robust Conversion of CCG Derivations to Phrase Structure Trees},
  author    = {Jonathan K. Kummerfeld and Dan Klein and James R. Curran},
  booktitle = {Proceedings of the 50th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
  year      = {2012},
  pages     = {105--109},
  month     = {July},
  location  = {Jeju Island, South Korea},
  software  = {https://jkk.name/berkeley-ccg2pst/},
  url       = {http://www.aclweb.org/anthology/P12-2021.pdf},
  slides    = {http://www.jkk.name/pub/acl12conversion_keynote.key},
  slidespdf = {http://www.jkk.name/pub/acl12conversion_slides.pdf},
}

      

We propose an improved, bottom-up method for converting CCG derivations into PTB-style phrase structure trees. In contrast with past work (Clark and Curran, 2009), which used simple transductions on category pairs, our approach uses richer transductions attached to single categories. Our conversion preserves more sentences under round-trip conversion (51.1% vs. 39.6%) and is more robust. In particular, unlike past methods, ours does not require ad-hoc rules over non-local features, and so can be easily integrated into a parser.

A Machine Learning Approach to Convert CCGbank to Penn Treebank, Xiaotian Zhang, Hai Zhao, Cong Hui, CoLing, 2012

@InProceedings{conll11coreference,
  title     = {Mention Detection: Heuristics for the OntoNotes annotations},
  author    = {Jonathan K. Kummerfeld and Mohit Bansal and David Burkett and Dan Klein},
  booktitle = {Proceedings of the Fifteenth Conference on Computational Natural Language Learning: Shared Task},
  year      = {2011},
  pages     = {102--106},
  month     = {June},
  location  = {Portland, Oregon, USA},
  url       = {http://www.aclweb.org/anthology/W11-1916.pdf},
  poster    = {http://www.jkk.name/pub/conll11coreference_poster.pdf},
}

      

Our submission was a reduced version of the system described in Haghighi and Klein (2010), with extensions to improve mention detection to suit the OntoNotes annotation scheme. Including exact matching mention detection in this shared task added a new and challenging dimension to the problem, particularly for our system, which previously used a very permissive detection method. We improved this aspect of the system by adding filters based on the annotation scheme for OntoNotes and analysis of system behavior on the development set. These changes led to improvements in coreference F-score of 10.06, 5.71, 6.78, 6.63 and 3.09 on the MUC, B3 , Ceaf-e, Ceaf-m and Blanc, metrics, respectively, and a final task score of 47.10.

Improving mention detection for Basque based on a deep error analysis, Ander Soraluze, Olatz Arregi, Xabier Arregi, Arantza Diaz de Ilarraza, Natural Language Engineering, 2016

Mention detection: First steps in the development of a Basque coreference resolution system, Ander Soraluze, Olatz Arregi, Xabier Arregi, Klara Ceberio, Arantza Diaz de Ilarraza, Proceedings of KONVENS, 2012

Detecting Apposition for Text Simplification in Basque, Itziar Gonzalez-Dios, Maria Jesus Aranzabe, Arantza Diaz de Ilarraza, Ander Soraluze, , 2013

Co-reference Resolution in Farsi Corpora, Maryam Nazaridoust, Behrouz Minaie Bidgoli, Siavash Nazaridoust, , 2014

Removing the Training Wheels: A Coreference Dataset that Entertains Humans and Challenges Computers, Anupam Guha, Mohit Iyyer, Danny Bouman, Jordan Boyd-Graber, NAACL, 2015

ARRAU: Linguistically-Motivated Annotation of Anaphoric Descriptions, Olga Uryupina, Ron Artstein, Antonella Bristot, Federica Cavicchio, Kepa J. Rodriguez, Massimo Poesio, LREC, 2016

A scaffolding approach to coreference resolution integrating statistical and rule-based models, Heeyoung Lee, Mihai Surdeanu, Dan Jurafsky, NLE, 2017

Apports des analyses syntaxiques pour la d'etection automatique de mentions dans un corpus de francais oral, Loic Grobol, Isabelle Tellier, Eric De La Clergerie, Marco Dinarelli, and Frederic Landragin, Actes de la 24e Conference sur le Traitement Automatique des Langues Naturelles (TALN), 2017

Mention Detection Using Pointer Networks for Coreference Resolution, Cheoneum Park, Changki Lee, Soojong Lim, ETRI Journal, 2017

@Article{prl10chemistry,
  title     = {Spatiotemporal Hierarchy of Relaxation Events, Dynamical Heterogeneities, and Structural Reorganization in a Supercooled Liquid},
  author    = {Raphael Candelier and Asaph Widmer-Cooper and Jonathan K. Kummerfeld and Olivier Dauchot and Giulio Biroli and Peter Harrowell and David R. Reichman},
  journal   = {Physical Review Letters},
  volume    = {105},
  number    = {13},
  pages     = {135702},
  doi       = {10.1103/PhysRevLett.105.135702},
  numpages  = {4},
  year      = {2010},
  month     = {September},
  publisher = {American Physical Society},
  url       = {http://prl.aps.org/abstract/PRL/v105/i13/e135702},
  arxiv     = {https://arxiv.org/abs/0912.0193},
}

      

We identify the pattern of microscopic dynamical relaxation for a two-dimensional glass-forming liquid. On short time scales, bursts of irreversible particle motion, called cage jumps, aggregate into clusters. On larger time scales, clusters aggregate both spatially and temporally into avalanches. This propagation of mobility takes place along the soft regions of the systems, which have been identified by computing isoconfigurational Debye-Waller maps. Our results characterize the way in which dynamical heterogeneity evolves in moderately supercooled liquids and reveal that it is astonishingly similar to the one found for dense glassy granular media.

Structural phases in non-additive soft-disk mixtures: Glasses, substitutional order, and random tilings A. Widmer-Cooper, P. Harrowell, Journal of Chemical Physics, 2011

Transient slowing down relaxation dynamics of the supercooled dusty plasma liquid after quenching Yen-Shuo Su, Chong-Wai Io, Lin I, Physical Review E, 2012

Finite Size Scaling of the Dynamical Free-Energy in a Kinetically Constrained Model Thierry Bodineau, Vivien Lecomte, Cristina Toninelli, Journal of Statistical Physics, 2012

Manifestations of dynamical facilitation in glassy materials Yael S. Elmatad, Aaron S. Keys, Physcial Review E, 2012

Trajectory entanglement in dense granular materials James G Puckett, Frederic Lechenault, Karen E Daniels, Jean-Luc Thiffeault, Journal of Statistical Mechanics: Theory and Experiment, 2012

Excitations Are Localized and Relaxation Is Hierarchical in Glass-Forming Liquids Aaron S. Keys, Lester O. Hedges, Juan P. Garrahan, Sharon C. Glotzer, David Chandler, Physical Review, 2011

Perspective: Supercooled liquids and glasses M. D. Ediger, Peter Harrowell, Journal of Chemical Physics, 2012

Gel formation and aging in weakly attractive nanocolloid suspensions at intermediate concentrations Hongyu Guo, S. Ramakrishnan, James L. Harden, Robert L. Leheny, Journal of Chemical Physics, 2011

Mean-field cage theory for the random close packed state of a metastable hard-sphere glass Xian-Zhi Wang, Physica A, 2012

Theoretical perspective on the glass transition and amorphous materials Ludovic Berthier, Giulio Biroli, Reviews of Modern Physics, 2011

Dynamical facilitation decreases when approaching the granular glass transition R. Candelier, O. Dauchot, G. Biroli, Europhysics Letters, 2010

Jammed Particle Configurations and Dynamics in High-Density Lennard-Jones Binary Mixtures in Two Dimensions Hayato Shiba, Akira Onuki, Progress of Theoretical Physics Supplement, 2010

Dynamic heterogeneities, boson peak, and activation volume in glass-forming liquids L. Hong, V. N. Novikov, A. P. Sokolov, Physical Review E, 2011

From Coupled Elementary Units to the Complexity of the Glass Transition Christian Rehwald, Oliver Rubner, Andreas Heuer, Physical Review Letters, 2010

Dynamics of thermal vibrational motions and stringlike jump motions in three-dimensional glass-forming liquids Takeshi Kawasaki, Akira Onuki, AIP: The Journal of Chemical Physics, 2013

Local elastic response measured near the colloidal glass transition D. Anderson, D. Schaar, H. G. E. Hentschel, J. Hay, Piotr Habdas, Eric R. Weeks, AIP: The Journal of Chemical Physics, 2013

Dynamical Heterogeneities in Glasses, Colloids, and Granular Media Ludovic Berthier, Giulio Biroli, Jean-Philippe Bouchaud, Luca Cipelletti, Wim van Saarloos, , 2011

Microscopic Picture of Cooperative Processes in Restructuring Gel Networks Jader Colombo, Asaph Widmer-Cooper, Emanuela Del Gado, Physical Review Letters, 2013

Distribution of local relaxation events in an aging three-dimensional glass: Spatiotemporal correlation and dynamical heterogeneity Anton Smessaert, Jorg Rottler, Physical Review E, 2013

Avalanches mediate crystallization in a hard-sphere glass Eduardo Sanz, Chantal Valeriani, Emanuela Zaccarelli, Wilson C. K. Poon, Michael E. Cates and Peter N. Pusey, Proceedings of the National Academy of Sciences, 2013

Distributions of single-molecule properties as tools for the study of dynamical heterogeneities in nanoconfined water G B Suffritti, P Demontis, J Gulin-Gonzalez, M Masia, Journal of Physics: Condensed Matter, 2014

Stress-induced microcracking and cooperative motion of cold dusty plasma liquids Chi Yang, Lin I, Physical Review E, 2014

Dynamics in a tetrahedral network glassformer: Vibrations, network rearrangements, and diffusion Takeshi Kawasaki, Kang Kim, Akira Onuki, The Journal of Chemical Physics, 2014

Order parameter for structural heterogeneity in disordered solids Hua Tong, Ning Xu, Physical Review E, 2014

Relaxation pathway confinement in glassy dynamics J. A. Rodriguez Fris, M. A. Frechero, G. A. Appignanesi, The Journal of Chemical Physics, 2014

Dynamical Heterogeneity in the Supercooled Liquid State of the Phase Change Material GeTe Gabriele Cesare Sosso, Jader Colombo, Joerg Behler, Emanuela Del Gado, Marco Bernasconi, The Journal of Physical Chemistry B, 2014

Flexible confinement leads to multiple relaxation regimes in glassy colloidal liquids Ian Williams, Erdal C. Oguz, Paul Bartlett, Hartmut Lowen, C. Patrick Royall, The Journal of Chemical Physics, 2015

Mutual information reveals multiple structural relaxation mechanisms in a model glass former Andrew J. Dunleavy, Karoline Wiesner, Ryoichi Yamamoto, C. Patrick Royall, Nature Communications, 2015

The role of local structure in dynamical arrest C. Patrick Royall, Stephen R. Williams, Physics Reports, 2015

Cooling the two-dimensional short spherocylinder liquid to the tetratic phase: Heterogeneous dynamics with one-way coupling between rotational and translational hopping Yen-Shuo Su, Lin I, Physical Review E, 2015

Computer simulation studies of the influence of side alkyl chain on glass transition behavior of carbazole trimer Chunyang Yu, Li Ma, Wei Huang, Yongfeng Zhou, Jingui Qin, Deyue Yan, Science China Chemistry, 2017

Structure-property relationships from universal signatures of plasticity in disordered solids E. D. Cubuk, R. J. S. Ivancic, S. S. Schoenholz, D. J. Strickl, A. Basu, Z. S. Davidson, J. Fontaine, J. L. Hor, Y.-R. Huang, Y. Jiang, N. C. Keim, K. D. Koshigan, J. A. Lefever, T. Liu, X.-G. Ma, D. J. Magagnosc, E. Morrow, C. P. Ortiz, J. M. Rieser, A. Shavit, T. Still, Y. Xu, Y. Zhang, K. N. Nordstrom, P. E. Arratia, R. W. Carpick, D. J. Durian, Z. Fakhraai, D. J. Jerolmack, Daeyeon Lee, Ju Li, R. Riggleman, K. T. Turner, A. G. Yodh, D. S. Gianola and rea J. Liu, Science, 2017

Continuous-time random-walk approach to supercooled liquids: Self-part of the van Hove function and related quantities J. Helfferich, J. Brisch, H. Meyer, O. Benzerara, F. Ziebert, J. Farago, J. Baschnagel, The European Physical Journal E, 2018

Heterogeneous Activation, Local Structure, and Softness in Supercooled Colloidal Liquids Xiaoguang Ma, Zoey S. Davidson, Tim Still, Robert J. S. Ivancic, S. S. Schoenholz, A. J. Liu and A. G. Yodh, Physical Review Letters, 2019

The Glassy Dynamics Predicted by the Mutual Role of the Free and Activation Volumes Wycliffe Kiprop Kipnusu, Mohamed Elsayed, Ciprian Iacob, Sebastian Pawlus, Reinhard Krause Rehberg, and Marian Paluch, Soft Matter, 2019

Multiscale Coherent Excitations in Microscopic Acoustic Wave Turbulence of Cold Dusty Plasma Liquids Hu, Hao-Wei and Wang, Wen and I, Lin, Physical Review Letters, 2019

Active particles sense micromechanical properties of glasses Celia Lozano, Juan Ruben Gomez-Solano, Clemens Bechinger, Nature Materials, 2019

@InProceedings{coling10morph,
  title     = {Morphological Analysis Can Improve a CCG Parser for English},
  author    = {Matthew Honnibal and Jonathan K. Kummerfeld and James R. Curran},
  booktitle = {Proceedings of the 23rd International Conference on Computational Linguistics},
  year      = {2010},
  pages     = {445--453},
  location  = {Beijing, China},
  month     = {August},
  url       = {http://www.aclweb.org/anthology/C10-2051.pdf},

We use morphological analysis to split verb inflectional suffixes into separate tokens, so that they can receive their own lexical categories. We find that this improves accuracy when the splits are based on correct POS tags, but that errors in gold standard or automatically assigned POS tags are costly for the system. This shows that the parser can benefit from morphological analysis, so long as the analysis is correct.},
}

      

Because English is a low morphology language, current statistical parsers tend to ignore morphology and accept some level of redundancy. This paper investigates how costly such redundancy is for a lexicalised grammar such as CCG.

We use morphological analysis to split verb inflectional suffixes into separate tokens, so that they can receive their own lexical categories. We find that this improves accuracy when the splits are based on correct POS tags, but that errors in gold standard or automatically assigned POS tags are costly for the system. This shows that the parser can benefit from morphological analysis, so long as the analysis is correct.

Morpho-syntactic Lexical Generalization for CCG Semantic Parsing, Adrienne Wang, Tom Kwiatkowski, Luke Zettlemoyer, EMNLP, 2014

Wide-Coverage Parsing, Semantics, and Morphology, Ruket Cakici, Mark Steedman, and Cem Bozsahin, , 2018

CCG Supertagging Using Morphological and Dependency Syntax Information, Ngoc Luyen Le, Yannis Haralambous, International Conference on Computational Linguistics and Intelligent Text Processing, 2019

@InProceedings{acl10adapt,
  title     = {Faster Parsing by Supertagger Adaptation},
  author    = {Jonathan K. Kummerfeld and Jessika Roesner and Tim Dawborn and James Haggerty and James R. Curran and Stephen Clark},
  booktitle = {Proceedings of the 48th Annual Meeting of the Association for Computational Linguistics},
  month     = {July},
  year      = {2010},
  location  = {Uppsala, Sweden},
  pages     = {345--355},
  software  = {http://downloads.schwa.org/acl10adapt_fast_news_model.tar.bz2},
  slidespdf = {http://www.jkk.name/pub/acl10adapt_slides.pdf},
  url       = {http://www.aclweb.org/anthology/P10-1036.pdf},
}

      

We propose a novel self-training method for a parser which uses a lexicalised grammar and supertagger, focusing on increasing the speed of the parser rather than its accuracy. The idea is to train the supertagger on large amounts of parser output, so that the supertagger can learn to supply the supertags that the parser will eventually choose as part of the highest scoring derivation. Since the supertagger supplies fewer supertags overall, the parsing speed is increased. We demonstrate the effectiveness of the method using a CCG supertagger and parser, obtaining significant speed increases on newspaper text with no loss in accuracy. We also show that the method can be used to adapt the CCG parser to new domains, obtaining accuracy and speed improvements for Wikipedia and biomedical text.

Chart pruning for fast lexicalised-grammar parsing, Yue Zhanga, Byung-Gyu Ahn, Stephen Clark, Curt Van Wyk, James R. Curran, Laura Rimell, CoLing, 2010

A Comparison of Loopy Belief Propagation and Dual Decomposition for Integrated CCG Supertagging and Parsing, Michael Auli, Adam Lopez, ACL, 2011

Efficient CCG Parsing: A* versus Adaptive Supertagging, Michael Auli, Adam Lopez, ACL, 2011

Exciting and interesting: issues in the generation of binomials Ann Copestake, Aurelie Herbelot, Proceedings of the UCNLG+Eval: Language Generation and Evaluation Workshop, 2011

Automatic recognition of conceptualization zones in scientific articles and two life science applications, Maria Liakata, Shyamasree Saha, Simon Dobnik, Colin Batchelor, Dietrich Rebholz-Schuhmann, Bioinformatics, 2012

Frontier Pruning for Shift-Reduce CCG Parsing, Stephen Merity, James Curran, ALTA, 2011

Ubertagging: Joint Segmentation and Supertagging for English, Rebecca Dridan, EMNLP, 2013

A* CCG Parsing with a Supertag-factored Model, Mike Lewis, Mark Steedman, EMNLP, 2014

CCG Supertagging with a Recurrent Neural Network, Wenduan Xu, Michael Auli, Stephen Clark, ACL, 2015

@InProceedings{alta09tagging,
  title     = {Faster parsing and supertagging model estimation},
  author    = {Jonathan K. Kummerfeld and Jessika Roesner and James R. Curran},
  booktitle = {Proceedings of the Australasian Language Technology Association Workshop 2009},
  year      = {2009},
  pages     = {62--70},
  location  = {Sydney, Australia},
  month     = {December},
  url       = {http://www.aclweb.org/anthology/U09-1009.pdf},
  slidespdf = {http://www.jkk.name/pub/alta09tagging_slides.pdf},

We demonstrate faster methods for training a supertagger using hundreds of millions of automatically annotated words, constructing statistical models that further constrain the number of derivations the parser must consider. By introducing new features and using an automatically annotated corpus we are able to double parsing speed on Wikipedia and the Wall Street Journal, and gain accuracy slightly when parsing Section 00 of the Wall Street Journal.},
}

      

Parsers are often the bottleneck for data acquisition, processing text too slowly to be widely applied. One way to improve the efficiency of parsers is to construct more confident statistical models. More training data would enable the use of more sophisticated features and also provide more evidence for current features, but gold standard annotated data is limited and expensive to produce.

We demonstrate faster methods for training a supertagger using hundreds of millions of automatically annotated words, constructing statistical models that further constrain the number of derivations the parser must consider. By introducing new features and using an automatically annotated corpus we are able to double parsing speed on Wikipedia and the Wall Street Journal, and gain accuracy slightly when parsing Section 00 of the Wall Street Journal.

@PhDThesis{thesis09adapt,
  title     = {Adaptive Supertagging for Faster Parsing},
  author    = {Jonathan K. Kummerfeld},
  school    = {The University of Sydney},
  year      = {2009},
  location  = {Sydney, Australia},
  url       = {http://www.jkk.name/pub/thesis09adapt_thesis.pdf},
  poster    = {http://www.jkk.name/pub/thesis09adapt_poster.pdf},
  slidespdf = {http://www.jkk.name/pub/thesis09adapt_slides.pdf},

  This work demonstrates several major novel ideas that improve parsing efficiency. The core idea is that the tags chosen by the parser are gold standard data for its supertagger. This leads to the second surprising conceptual development, that decreasing tagging accuracy can improve parsing performance. To demonstrate these ideas required extensive development of the C&C supertagger, including imple- mentation of more efficient estimation algorithms and parallelisation of the training process. This was particularly challenging as the C&C supertagger is a state-of-the-art high performance system designed with a focus on speed rather than flexibility.

  I was able to significantly improve performance on the standard evaluation corpus by using the parser to generate extremely large new resources for supertagger training. I have also shown that these methods provide significant benefits on another domain, Wikipedia text, without the cost of generating human annotated data sets. These parsing performance gains occur while supertagging accuracy decreases.

  Despite extensive use of supertaggers to improve parsing efficiency there has been no comprehensive study of the interaction between a supertagger and a parser. I present the first systematic exploration of the relationship, show the potential benefits of understanding it, and demonstrate a novel algorithm for optimising the parameters that define it.

  I have constructed models that process newspaper text 86% faster than previously, and Wikipedia text 30% faster, without any loss in accuracy and without the aid of extra gold standard resources in either domain. This work will lead directly to improvements in a range of Natural Language Processing tasks by enabling the use of far more parsed data.},
}

      

Statistical parsers are crucial for tackling the grand challenges of Natural Language Processing. The most effective approaches to these tasks are data driven, but parsers are too slow to be effectively used on large data sets. State-of-the-art parsers generally cannot process more than one sentence a second, and the fastest cannot process more than fifty sentences a second. The situation is even worse when they are applied outside of the domain of their training data. The fastest systems have two components, a parser, which has time complexity O(n3) and a supertagger, which has linear time complexity. By shifting work from the parser to the supertagger we dramatically improve speed.

This work demonstrates several major novel ideas that improve parsing efficiency. The core idea is that the tags chosen by the parser are gold standard data for its supertagger. This leads to the second surprising conceptual development, that decreasing tagging accuracy can improve parsing performance. To demonstrate these ideas required extensive development of the C&C supertagger, including imple- mentation of more efficient estimation algorithms and parallelisation of the training process. This was particularly challenging as the C&C supertagger is a state-of-the-art high performance system designed with a focus on speed rather than flexibility.

I was able to significantly improve performance on the standard evaluation corpus by using the parser to generate extremely large new resources for supertagger training. I have also shown that these methods provide significant benefits on another domain, Wikipedia text, without the cost of generating human annotated data sets. These parsing performance gains occur while supertagging accuracy decreases.

Despite extensive use of supertaggers to improve parsing efficiency there has been no comprehensive study of the interaction between a supertagger and a parser. I present the first systematic exploration of the relationship, show the potential benefits of understanding it, and demonstrate a novel algorithm for optimising the parameters that define it.

I have constructed models that process newspaper text 86% faster than previously, and Wikipedia text 30% faster, without any loss in accuracy and without the aid of extra gold standard resources in either domain. This work will lead directly to improvements in a range of Natural Language Processing tasks by enabling the use of far more parsed data.

@TechReport{report09jhu,
  title     = {Large-Scale Syntactic Processing: Parsing the Web},
  author    = {Stephen Clark and Ann Copestake and James R. Curran and Yue Zhang and Aurelie Herbelot and James Haggerty and Byung-Gyu Ahn and Curt Van Wyk and Jessika Roesner and Jonathan K. Kummerfeld and Tim Dawborn},
  year      = {2009},
  institution = {Johns Hopkins University},
  url       = {http://www.jkk.name/pub/report09jhu.pdf},

The goals of our workshop were to scale and adapt an existing wide-coverage parser to Wikipedia text; improve the efficiency of the parser through various methods of chart pruning; use self-training to improve the efficiency and accuracy of the parser; use the parsed wiki data for an innovative form of bootstrapping to make the parser both more efficient and more accurate; and finally use the parsed web data for improved disambiguation of coordination structures, using a variety of syntactic and semantic knowledge sources.

The focus of the research was the C&C parser (Clark and Curran, 2007c), a state-of-the-art statistical parser based on Combinatory Categorial Grammar (ccg). The parser has been evaluated on a number of standard test sets achieving state-of-the-art accuracies. It has also recently been adapted successfully to the biomedical domain (Rimell and Clark, 2009). The parser is surprisingly efficient, given its detailed output, processing tens of sentences per second. For web-scale text processing, we aimed to make the parser an order of magnitude faster still. The C&C parser is one of only very few parsers currently available which has the potential to produce detailed, accurate analyses at the scale we were considering.},
}

      

Scalable syntactic processing will underpin the sophisticated language technology needed for next generation information access. Companies are already using nlp tools to create web-scale question answering and ‘semantic search’ engines. Massive amounts of parsed web data will also allow the automatic creation of semantic knowledge resources on an unprecedented scale. The web is a challenging arena for syntactic parsing, because of its scale and variety of styles, genres, and domains.

The goals of our workshop were to scale and adapt an existing wide-coverage parser to Wikipedia text; improve the efficiency of the parser through various methods of chart pruning; use self-training to improve the efficiency and accuracy of the parser; use the parsed wiki data for an innovative form of bootstrapping to make the parser both more efficient and more accurate; and finally use the parsed web data for improved disambiguation of coordination structures, using a variety of syntactic and semantic knowledge sources.

The focus of the research was the C&C parser (Clark and Curran, 2007c), a state-of-the-art statistical parser based on Combinatory Categorial Grammar (ccg). The parser has been evaluated on a number of standard test sets achieving state-of-the-art accuracies. It has also recently been adapted successfully to the biomedical domain (Rimell and Clark, 2009). The parser is surprisingly efficient, given its detailed output, processing tens of sentences per second. For web-scale text processing, we aimed to make the parser an order of magnitude faster still. The C&C parser is one of only very few parsers currently available which has the potential to produce detailed, accurate analyses at the scale we were considering.

Introducing More Features to Improve Chinese Shift-Reduce Parsing Hongxian Wang, Qiang Zhou, Liou Chen, Asia Pacific Signal and Information Processing Association, Annual Summit and Conference, 2011

Probabilistic models of similarity in syntactic context Diarmuid Seaghdha, Anna Korhonen, EMNLP, 2011

Evaluation Reportof the third Chinese Parsing Evaluation: CIPSSIGHAN-ParsEval-2012 Qiang Zhou, Proceedings of the Second CIPS-SIGHAN Joint Conference on Chinese Language Processing, 2012

Evaluation Reportof the third Chinese Parsing Evaluation: CIPSSIGHAN-ParsEval-2014 Qiang Zhou, Proceedings of the Third CIPS-SIGHAN Joint Conference on Chinese Language Processing, 2014

Interpreting compound nouns with kernel methods Diarmuid Seaghdha, Ann Copestake, Natural Language Engineering, 2013

CYK-based Decision and Forest Parsing Algorithm for Combinatory Categorial Grammar Qingjiang Wang, Zhengping Wang, Lin Zhang, Journal of Information and Computational Science, 2014

A Streaming Dataflow Implementation of Parallel Cocke–Younger–Kasami Parser D. Bojic, M. Bojovic, , 2016

Leveraging a Semantically Annotated Corpus to Disambiguate Prepositional Phrase Attachment Guy Emerson, Ann Copestake, Proceedings of the 11th International Conference on Computational Semantics, 2015

How important is syntactic parsing accuracy? An empirical evaluation on rule-based sentiment analysis, Carlos Gomez-Rodriguez, Iago Alonso-Alonso, David Vilares, Artificial Intelligence Review, 2017

@InProceedings{alta08vpc,
  title     = {Classification of Verb Particle Constructions with the Google Web1T Corpus},
  author    = {Jonathan K. Kummerfeld and James R. Curran},
  booktitle = {Proceedings of the Australasian Language Technology Association Workshop 2008},
  month     = {December},
  year      = {2008},
  location  = {Hobart, Australia},
  pages     = {55--63},
  volume    = {6},
  url       = {http://www.aclweb.org/anthology/U08-1008.pdf},
  poster    = {http://www.jkk.name/pub/alta08vpc_poster.eps},
}

      

Manually maintaining comprehensive databases of multi-word expressions, for example Verb-Particle Constructions (VPCs), is infeasible. We describe a new classifier for potential VPCs, which uses information in the Google Web1T corpus to perform a simple linguistic constituency test. Specifically, we consider the fronting test, comparing the frequencies of the two possible orderings of the given verb and particle. Using only a small set of queries for each verb-particle pair, the system was able to achieve an F-score of 78.4% in our evaluation while processing thousands of queries a second.

New Tools for Web-Scale N-grams, Dekang Lin, Kenneth Church, Heng Ji, Satoshi Sekine, David Yarowsky, Shane Bergsma, Kailash Patil, Emily Pitler, Rachel Lathbury, Vikram Rao, Kapil Dalwani, Sushant Narsale, LREC, 2010

Corpus-based Extraction of Japanese Compound Verbs, James Breen, Timothy Baldwin, ALTA, 2009

Predicting the Semantic Compositionality of Prefix Verbs, Shane Bergsma, Aditya Bhargava, Hua He, Grzegorz Kondrak, EMNLP, 2010

Practical Linguistic Steganography Using Contextual Synonym Substitution and Vertex Colour Coding, Ching-Yun Chang, Stephen Clark, EMNLP, 2010

POS Tagging of English Particles for Machine Translation, Jianjun Ma, Degen Huang, Haixia Liu, Wenfeng Sheng, MT Summit of the International Association for Machine Translation, 2011

Automatic Classification of German an Particle Verbs, Sylvia Springorum, Sabine Schulte im Walde, Antje Roßdeutscher, LREC, 2012

Practical Linguistic Steganography using Contextual Synonym Substitution and a Novel Vertex Coding Method, Ching-Yun Chang, Stephen Clark, CL, 2014

@Article{chem08packing,
  title     = {The densest packing of AB binary hard-sphere homogeneous compounds across all size ratios},
  author    = {Jonathan K. Kummerfeld and Toby S Hudson and Peter Harrowell},
  journal   = {The Journal of Physical Chemistry B},
  month     = {August},
  year      = {2008},
  volume    = {112},
  issue     = {35},
  pages     = {10773--10776},
  url       = {http://pubs.acs.org/doi/abs/10.1021/jp804953r},
}

      

This paper considers the homogeneous packing of binary hard spheres in an equimolar stoichiometry, and postulates the densest packing at each sphere size ratio. Monte Carlo simulated annealing optimizations are seeded with all known atomic inorganic crystal structures, and the search is performed within the degrees of freedom associated with each homogeneous AB structure type. Structures isopointal to the FeB structure type are found to have the highest packing fraction at all sphere size ratios. The optimized structures match or improve on the best previously demonstrated packings of this type, and show that compound structures can pack more densely than segregated close-packed structures at all radius ratios less than 0.62.

Phase Diagram and Structural Diversity of the Densest Binary Sphere Packings Adam B. Hopkins, Yang Jiao, Frank H. Stillinger, Salvatore Torquato, Physical Review Letters, 2011

Densest binary sphere packings Adam B. Hopkins, Frank H. Stillinger, Salvatore Torquato, Physical Review E, 2012

Geometrical Frustration and Static Correlations in a Simple Glass Former Benoit Charbonneau, Patrick Charbonneau, Gilles Tarjus, Physical Review Letters, 2012

Structural phases in non-additive soft-disk mixtures: Glasses, substitutional order, and random tilings Asaph Widmer-Cooper, Peter Harrowell, Journal of Chemical Physics, 2011

Phase diagram of hard snowman-shaped particles Matthew Dennison, Kristina Milinkovi ́c, Marjolein Dijkstra, Journal of Chemical Physics, 2012

Dense Sphere Packing in the NaZn13 Structure Type Toby S. Hudson, The Journal of Physical Chemistry C, 2010

Electrophoretic deposition of binary energetic composites Kyle Thomas Sullivan, Marcus Andre Worsley, Joshua David Kuntz, Alex Eydmann Gash, Combustion and Flame, 2012

Multicomponent periodic nanoparticle superlattices Paul Podsiadlo, Galyna Krylova, Arnaud Demortiere, Elena Shevchenko, Journal of Nanoparticle Research, 2011

Structural searches using isopointal sets as generators: densest packings for binary hard sphere mixtures Toby S. Hudson, Peter Harrowell, Journal of Physics: Condensed Matter, 2011

Prediction of binary hard-sphere crystal structures Laura Filion, Marjolein Dijkstra, Physical Review E, 2009

Efficient Method for Predicting Crystal Structures at Finite Temperature: Variable Box Shape Simulations Laura Filion, Matthieu Marechal, Bas van Oorschot, Daniel Pelt, Frank Smallenburg, Marjolein Dijkstra, Physical Review Letters, 2009

Dense Packings of Hard Spheres of Different Sizes Based on Filling Interstices in Uniform Three-Dimensional Tilings Toby S. Hudson, Peter Harrowell, The Journal of Physical Chemistry B, 2008

Crystal nucleation in binary hard-sphere mixtures: the effect of order parameter on the cluster composition Ran Ni, Frank Smallenburg, Laura Filion, Marjolein Dijkstra, Molecular Physics, 2011

New High Density Packings of Similarly-Sized Binary Spheres Patrick I. O'Toole, Toby S. Hudson, The Journal of Physical Chemistry C, 2011

Structural search for dense packing of concave and convex shapes in two dimensions Nabiha T Elias, Toby S Hudson, Journal of Physics: Conference Series, 2012

On the Phase Behavior of Binary Mixtures of Nanoparticles Avi Ben-Simon, Hagai Eshet, Eran Rabani, ACS Nano, 2013

Favoured Local Structures in Liquids and Solids: a 3D Lattice Model Pierre Ronceray, Peter Harrowell, The Self Journal of Science, 2015

Packing concave molecules in crystals and amorphous solids: on the connection between shape and local structure Cerridwen Jennings, Malcolm Ramsay, Toby Hudson, Peter Harrowell, Molecular Physics, 2015

Binary nanoparticle superlattices of soft-particle systems Alex Travesset, Proceedings of the National Academy of Sciences, 2015

A Geometric-Structure Theory for Maximally Random Jammed Packings Jianxiang Tian, Yaopengxiao Xu, Yang Jiao, Salvatore Torquato, Scientific Reports, 2015

Perspective: Basic understanding of condensed phases of matter via packing models, Salvatore Torquato, The Journal of Chemical Physics, 2018

Using symmetry to elucidate the importance of stoichiometry in colloidal crystal assembly, Nathan A. Mahynski, Evan Pretti, Vincent K. Shen, Jeetain Mittal, Nature Communicationsvolume, 2019