The answer is wp_get_referer(). This function is working inside ajax_query_attachments_args. I can then use get_post_type with the post id from url parameters
$referer = parse_url(wp_get_referer());
parse_str($referer['query'], $params);
if (isset($params['post'])){
$post_type = get_post_type($params['post']);
} else if (strpos($referer['path'], 'post-new.php') !== false && !isset($params['post_type'])){
$post_type="post";
} else {
$post_type="";
}
if ( $post_type == 'post' ) {
$query['tax_query'] = array(
array(
'taxonomy' => 'category',
'operator' => 'NOT EXISTS'
),
);
}