I've finally solved by building directly the accessoryView, by creating UIButtons (and no more buttomItems) and using config.
Here a code extract for those interested:
let letterBack = UIImage(named: "letterBackground")!
if #available(iOS 26.0, *) {
let targetSize = CGSize(width: 40, height: 40)
let scaledImage = letterBack.scalePreservingAspectRatio(targetSize: targetSize)
var config = UIButton.Configuration.filled()
config.background.image = scaledImage
config.background.imageContentMode = .scaleAspectFill
let bar = UIView(frame:CGRect(x: 0, y: 0, width: 200, height: 44)) ?? create bar as UIView and not UIToolbar
config.baseForegroundColor = .black
config.titleAlignment = .center
// set each button
config.title = "I"
let letterI = UIButton(configuration: config)
letterI.frame = CGRect(x: 5, y: 0, width: 40, height: 36)
letterI.addTarget(self, action: #selector(letterButtonTapped(_:)), for: .touchUpInside)
letterI.tag = 1. // Use the tag to know which button was tapped
bar.addSubview(letterI)
// the same for other buttons
// set the UIView as the accessory
answerTextField.inputAccessoryView = bar
}