@article{pullar-strecker2022hitting,

title = {Hitting the Target: Stopping Active Learning at the Cost-Based Optimum},

author = {Zac Pullar-Strecker and Katharina Dost and Eibe Frank and J\"{o}rg Wicker},

url = {https://arxiv.org/abs/2110.03802},

year  = {2022},

date = {2022-12-01},

urldate = {2021-10-08},

abstract = {Active learning allows machine learning models to be trained using fewer labels while retaining similar performance to traditional fully supervised learning. An active learner selects the most informative data points, requests their labels, and retrains itself. While this approach is promising, it leaves an open problem of how to determine when the model is `good enough' without the additional labels required for traditional evaluation. In the past, different stopping criteria have been proposed aiming to identify the optimal stopping point. However, optimality can only be expressed as a domain-dependent trade-off between accuracy and the number of labels, and no criterion is superior in all applications. This paper is the first to give actionable advice to practitioners on what stopping criteria they should use in a given real-world scenario. We contribute the first large-scale comparison of stopping criteria, using a cost measure to quantify the accuracy/label trade-off, public implementations of all stopping criteria we evaluate, and an open-source framework for evaluating stopping criteria. Our research enables practitioners to substantially reduce labelling costs by utilizing the stopping criterion which best suits their domain.},

keywords = {},

pubstate = {forthcoming},

tppubtype = {article}

}

@article{bensemann2022from,

title = {From What You See to What We Smell: Linking Human Emotions to Biomarkers in Breath},

author = {Joshua Bensemann and Hasnain Cheena and David Tse Juang Huang and Elizabeth Broadbendt and Jonathan Williams and J\"{o}rg Wicker},

year  = {2022},

date = {2022-12-01},

abstract = {Breath collection is a non-invasive method for monitoring biological processes occurring within the human body. Prior studies have extended these methods to observe the general processes occurring in groups of humans and are able to link them to what those groups are collectively experiencing. However, previous work lacked an objective measure of emotional stimuli. In this research, we applied machine learning techniques to breath data collected from cinema audiences to find associations between the biomarkers in the crowd's breath and both the audio-visual stimuli and thematic events of the movie.

This analysis enabled us to make direct links between what the group was experiencing and their biological response to that experience. To achieve this, we first extracted visual and auditory features from a movie and compared it to the biomarkers in the crowd's breath, using both regression and pattern mining techniques. Our results supported the theory that a crowd's collective experience has a direct correlation to the biomarkers in the crowd's breath. Consequently, these findings suggest that visual and auditory experiences have predictable effects on the human body that can be monitored without the requirement of expensive and/or invasive neuroimaging techniques.},

keywords = {},

pubstate = {forthcoming},

tppubtype = {article}

}

@article{Dost2022combatting,

title = {Combatting Over-Specialization Bias in Growing Chemical Databases},

author = {Katharina Dost and Zac Pullar-Strecker and Liam Brydon and Kunyang Zhang and Jasmin Hafner and Pat Riddle and J\"{o}rg Wicker},

doi = {https://doi.org/10.21203/rs.3.rs-2133331/v1},

year  = {2022},

date = {2022-10-05},

journal = {Research Square},

abstract = {Background: Predicting in advance the behavior of new chemical compounds can support the design process of new products by directing the research towards the most promising candidates and ruling out others. Such predictive models can be data-driven using Machine Learning or based on researchers' experience and depend on the collection of past results. In either case: models (or researchers) can only make reliable assumptions on compounds that are similar to what they have seen before. Therefore, consequent usage of these predictive models shapes the dataset and causes a continuous specialization shrinking the applicability domain of all trained models on this dataset in the future, and increasingly harming model-based exploration of the space. 



Proposed Solution: In this paper, we propose CANCELS (CounterActiNg Compound spEciaLization biaS), a technique that helps to break the dataset specialization spiral. Aiming for a smooth distribution of the compounds in the dataset, we identify areas in the space that fall short and suggest additional experiments that help bridge the gap. Thereby, we generally improve the dataset quality in an entirely unsupervised manner and create awareness of potential flaws in the data. CANCELS does not aim to cover the entire compound space and hence retains a desirable degree of specialization to a specified research domain. 



Results: An extensive set of experiments on the use-case of biodegradation pathway prediction not only reveals that the bias spiral can indeed be observed but also that CANCELS produces meaningful results. Additionally, we demonstrate that mitigating the observed bias is crucial as it cannot only intervene with the continuous specialization process, but also significantly improves a predictor's performance while reducing the amount of required experiments. Overall, we believe that CANCELS can support researchers in their experimentation process to not only better understand their data and potential flaws, but also to grow the dataset in a sustainable way. All code is available under github.com/KatDost/Cancels.},

note = {preprint},

keywords = {},

pubstate = {forthcoming},

tppubtype = {article}

}

@article{nokey,

title = {Development of a Seismic Loss Prediction Model for Residential Buildings using Machine Learning \textendash Christchurch, New Zealand},

author = {Samuel Roeslin and Quincy Ma and and Pavan Chigullapally and J\"{o}rg Wicker and Liam Wotherspoon},

doi = {10.5194/nhess-2022-227},

year  = {2022},

date = {2022-09-01},

urldate = {2022-09-01},

journal = {Natural Hazards and Earth System Sciences},

abstract = {This paper presents a new framework for the seismic loss prediction of residential buildings in Christchurch, New Zealand. It employs data science techniques, geospatial tools, and machine learning (ML) trained on insurance claims data from the Earthquake Commission (EQC) collected following the 2010\textendash2011 Canterbury Earthquake Sequence (CES). The seismic loss prediction obtained from the ML model is shown to outperform the output from existing risk analysis tools for New Zealand for each of the main earthquakes of the CES. In addition to the prediction capabilities, the ML model delivered useful insights into the most important features contributing to losses during the CES. ML correctly highlighted that liquefaction significantly influenced buildings losses for the 22 February 2011 earthquake. The results are consistent with observations, engineering knowledge, and previous studies, confirming the potential of data science and ML in the analysis of insurance claims data and the development of seismic loss prediction models using empirical loss data.},

note = {in review},

keywords = {},

pubstate = {forthcoming},

tppubtype = {article}

}

@article{tam2021holisticb,

title = {Holistic Evaluation of Biodegradation Pathway Prediction: Assessing Multi-Step Reactions and Intermediate Products},

author = {Jason Tam and Tim Lorsbach and Sebastian Schmidt and J\"{o}rg Wicker},

url = {https://jcheminf.biomedcentral.com/articles/10.1186/s13321-021-00543-x

https://chemrxiv.org/articles/preprint/Holistic_Evaluation_of_Biodegradation_Pathway_Prediction_Assessing_Multi-Step_Reactions_and_Intermediate_Products/14315963

https://dx.doi.org/10.26434/chemrxiv.14315963},

doi = {10.1186/s13321-021-00543-x},

year  = {2021},

date = {2021-09-03},

urldate = {2021-09-03},

journal = {Journal of Cheminformatics},

volume = {13},

number = {1},

pages = {63},

abstract = {The prediction of metabolism and biotransformation pathways of xenobiotics is a highly desired tool in environmental sciences, drug discovery, and (eco)toxicology. Several systems predict single transformation steps or complete pathways as series of parallel and subsequent steps. Their performance is commonly evaluated on the level of a single transformation step. Such an approach cannot account for some specific challenges that are caused by specific properties of biotransformation experiments. That is, missing transformation products in the reference data that occur only in low concentrations, e.g. transient intermediates or higher-generation metabolites. Furthermore, some rule-based prediction systems evaluate the performance only based on the defined set of transformation rules. Therefore, the performance of these models cannot be directly compared. In this paper, we introduce a new evaluation framework that extends the evaluation of biotransformation prediction from single transformations to whole pathways, taking into account multiple generations of metabolites. We introduce a procedure to address transient intermediates and propose a weighted scoring system that acknowledges the uncertainty of higher-generation metabolites. We implemented this framework in enviPath and demonstrate its strict performance metrics on predictions of in vitro biotransformation and degradation of xenobiotics in soil. Our approach is model-agnostic and can be transferred to other prediction systems. It is also capable of revealing knowledge gaps in terms of incompletely defined sets of transformation rules.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{chang2021intriguing,

title = {Intriguing Usage of Applicability Domain: Lessons from Cheminformatics Applied to Adversarial Learning},

author = {Luke Chang and Katharina Dost and Kaiqi Zhai and Ambra Demontis and Fabio Roli and Gillian Dobbie and J\"{o}rg Wicker},

url = {https://arxiv.org/abs/2105.00495},

year  = {2021},

date = {2021-05-02},

journal = {arxiv},

volume = {2105.00495},

abstract = {Defending machine learning models from adversarial attacks is still a challenge: none of the robust models is utterly immune to adversarial examples to date. Different defences have been proposed; however, most of them are tailored to particular ML models and adversarial attacks, therefore their effectiveness and applicability are strongly limited. A similar problem plagues cheminformatics: Quantitative Structure-Activity Relationship (QSAR) models struggle to predict biological activity for the entire chemical space because they are trained on a very limited amount of compounds with known effects. This problem is relieved with a technique called Applicability Domain (AD), which rejects the unsuitable compounds for the model. Adversarial examples are intentionally crafted inputs that exploit the blind spots which the model has not learned to classify, and adversarial defences try to make the classifier more robust by covering these blind spots. There is an apparent similarity between AD and adversarial defences. Inspired by the concept of AD, we propose a multi-stage data-driven defence that is testing for: Applicability: abnormal values, namely inputs not compliant with the intended use case of the model; Reliability: samples far from the training data; and Decidability: samples whose predictions contradict the predictions of their neighbours. It can be applied to any classification model and is not limited to specific types of adversarial attacks. With an empirical analysis, this paper demonstrates how Applicability Domain can effectively reduce the vulnerability of ML models to adversarial examples. },

note = {preprint},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{stepisnik2021comprehensive,

title = {A comprehensive comparison of molecular feature representations for use in predictive modeling},

author = {Toma\v{z} Stepi\v{s}nik and Bla\v{z} \v{S}krlj and J\"{o}rg Wicker and Dragi Kocev},

url = {http://www.sciencedirect.com/science/article/pii/S001048252030528X},

doi = {10.1016/j.compbiomed.2020.104197},

issn = {0010-4825},

year  = {2021},

date = {2021-03-01},

journal = {Computers in Biology and Medicine},

volume = {130},

pages = {104197},

abstract = {Machine learning methods are commonly used for predicting molecular properties to accelerate material and drug design. An important part of this process is deciding how to represent the molecules. Typically, machine learning methods expect examples represented by vectors of values, and many methods for calculating molecular feature representations have been proposed. In this paper, we perform a comprehensive comparison of different molecular features, including traditional methods such as fingerprints and molecular descriptors, and recently proposed learnable representations based on neural networks. Feature representations are evaluated on 11 benchmark datasets, used for predicting properties and measures such as mutagenicity, melting points, activity, solubility, and IC50. Our experiments show that several molecular features work similarly well over all benchmark datasets. The ones that stand out most are Spectrophores, which give significantly worse performance than other features on most datasets. Molecular descriptors from the PaDEL library seem very well suited for predicting physical properties of molecules. Despite their simplicity, MACCS fingerprints performed very well overall. The results show that learnable representations achieve competitive performance compared to expert based representations. However, task-specific representations (graph convolutions and Weave methods) rarely offer any benefits, even though they are computationally more demanding. Lastly, combining different molecular feature representations typically does not give a noticeable improvement in performance compared to individual feature representations.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}