You probably need to allow the mime types for them to be allowed to upload.
https://www.robertwent.com/blog/adding-custom-mime-types-for-wordpress-uploads/
//The following goes in a themes functions file or a custom hooks plugin
function so_387865_custom_upload_mimes ( $existing_mimes ) {
$existing_mimes['epub'] = 'application/epub+zip';
$existing_mimes['mobi'] = 'application/x-mobipocket-ebook';
return $existing_mimes;
}
add_filter('upload_mimes', 'so_387865_custom_upload_mimes');