Modify custom field input data before creating a new post

This runs your function right after the post has been saved (created/updated):

add_action('save_post','clean_data_while_saving_post',1);

function clean_data_while_saving_post($post_id) {
  $data = get_post_meta($post_id,'book_isbn',true);
  update_post_meta($post_id,'book_isbn',clean_isbn($data));
}