How to wrap WordPress image captions inside H2, H3 tags?

You can hook into the filter img_caption_shortcode and replace the whole captioned image. Here I’ve copied the caption shortcode function from WP4.5, left the version used if your theme declares HTML5 support as it is (using figcaption) and modified the non-HTML5 version to use h2.

function wpse_233354_img_caption_shortcode( $empty, $attr, $content = null ) {
    // New-style shortcode with the caption inside the shortcode with the link and image tags.
    if ( ! isset( $attr['caption'] ) ) {
        if ( preg_match( '#((?:<a [^>]+>\s*)?<img [^>]+>(?:\s*</a>)?)(.*)#is', $content, $matches ) ) {
            $content = $matches[1];
            $attr['caption'] = trim( $matches[2] );
        }
    } elseif ( strpos( $attr['caption'], '<' ) !== false ) {
        $attr['caption'] = wp_kses( $attr['caption'], 'post' );
    }

    $atts = shortcode_atts( array(
        'id'      => '',
        'align'   => 'alignnone',
        'width'   => '',
        'caption' => '',
        'class'   => '',
    ), $attr, 'caption' );

    $atts['width'] = (int) $atts['width'];
    if ( $atts['width'] < 1 || empty( $atts['caption'] ) )
        return $content;

    if ( ! empty( $atts['id'] ) )
        $atts['id'] = 'id="' . esc_attr( sanitize_html_class( $atts['id'] ) ) . '" ';

    $class = trim( 'wp-caption ' . $atts['align'] . ' ' . $atts['class'] );

    $html5 = current_theme_supports( 'html5', 'caption' );
    // HTML5 captions never added the extra 10px to the image width
    $width = $html5 ? $atts['width'] : ( 10 + $atts['width'] );

    /**
     * Filter the width of an image's caption.
     *
     * By default, the caption is 10 pixels greater than the width of the image,
     * to prevent post content from running up against a floated image.
     *
     * @since 3.7.0
     *
     * @see img_caption_shortcode()
     *
     * @param int    $width    Width of the caption in pixels. To remove this inline style,
     *                         return zero.
     * @param array  $atts     Attributes of the caption shortcode.
     * @param string $content  The image element, possibly wrapped in a hyperlink.
     */
    $caption_width = apply_filters( 'img_caption_shortcode_width', $width, $atts, $content );

    $style="";
    if ( $caption_width )
        $style="style="width: " . (int) $caption_width . 'px" ';

    $html="";
    if ( $html5 ) {
        $html="<figure " . $atts['id'] . $style . 'class="' . esc_attr( $class ) . '">'
        . do_shortcode( $content ) . '<figcaption class="wp-caption-text">' . $atts['caption'] . '</figcaption></figure>';
    } else {
        $html="<div " . $atts['id'] . $style . 'class="' . esc_attr( $class ) . '">'
        . do_shortcode( $content ) . '<h2 class="wp-caption-text">' . $atts['caption'] . '</h2></div>';
    }

    return $html;
}

add_filter( 'img_caption_shortcode', 'wpse_233354_img_caption_shortcode', 10, 3 );