Subsurface {Modeling} from {Borehole} {Descriptions} via {Transformer}-{Based} {Sequence} {Labeling}: {Contextual} {Classification} and {Limits} of {MC}-{Dropout} {Uncertainty}.

@inproceedings{Cilli2026RMa,
 abstract = {Geological borehole descriptions are a fundamental resource for subsurface modelling, yet they are commonly stored as heterogeneous and unstructured text, limiting their use in automated analysis. Existing geological natural language processing approaches often classify individual descriptions independently, overlooking the vertical context of stratigraphic successions. Moreover, high classification performance does not necessarily imply that model confidence and uncertainty remain reliable under spatial distribution shift. We present a context-aware sequence-labelling framework that combines pre-trained Sentence-BERT embeddings with a lightweight Transformer encoder to classify lithostratigraphic units from ordered borehole descriptions. The approach is evaluated on 95 manually digitized boreholes collected during 11 investigation campaigns at the Pianello hillslope in Southern Italy. Compared with a random forest baseline using static Sentence-BERT embeddings, contextual sequence modelling improves classification accuracy by approximately 13%, demonstrating the value of vertical stratigraphic context. Beyond classification performance, we assess predictive reliability using confidence and entropy-based uncertainty components derived from Monte Carlo dropout. The model is evaluated under leave-one-borehole-out validation, increasing spatial exclusion buffers, and controlled perturbations of semantic and positional features. Confidence is associated with prediction correctness under ordinary validation conditions. However, as local spatial support is removed, accuracy and Cohen’s kappa deteriorate more clearly than the MC-dropout-based epistemic uncertainty proxy increases. Controlled perturbations produce a similar mismatch between performance degradation and uncertainty response. These results show that lightweight contextual language models can support lithostratigraphic interpretation from heterogeneous borehole descriptions. At the same time, confidence and MC-dropout-derived uncertainty should not be treated as standalone guarantees of spatial reliability. Spatially aware validation remains essential for trustworthy geological machine learning.},
 author = {Cilli, Roberto and Clausel, Marianne},
 booktitle = {2026 {RING} meeting},
 language = {en},
 pages = {7--22},
 publisher = {ASGA},
 title = {Subsurface {Modeling} from {Borehole} {Descriptions} via {Transformer}-{Based} {Sequence} {Labeling}: {Contextual} {Classification} and {Limits} of {MC}-{Dropout} {Uncertainty}.},
 year = {2026}
}