pages shortcode filtering by category

ob_start() This function will turn output buffering on. While output
buffering is active no output is sent from the script (other than
headers), instead the output is stored in an internal buffer.

ob_get_clean() Gets the current buffer contents and delete current
output buffer.

try this code

function qut_research_subpages_shortcode_frontend($atts)
{
    $args = array(
        'post_type' => 'page',
        'posts_per_page' => '-1',
    );

    if (!empty($atts['category'])) {
        $args['pages_category'] = $atts['category'];
    }

    $theme_uri = get_template_directory_uri();

    $loop = new \WP_Query($args);
    $post_featured_image = get_the_post_thumbnail_url($loop>the_post(),'thumbnail');

    if ( $atts['layout'] == 'boxes' || $atts['layout'] == '' ) {

        ob_start(); ?>
        <div class="row" style="margin-top: 20px">

        <?php while ($loop->have_posts()) : $loop->the_post(); ?>

                    <div class="col-xl-2 col-lg-4 col-md-4 col-sm-4 col-xs-6" style="padding: 5px;">
                        <a class="project-subpage" href="https://wordpress.stackexchange.com/questions/284566/<?= the_permalink(); ?>">
                            <figure class="project-box" style="height: 120px; position: relative; background-size: cover; background-image: url(
                            <?php
                            if (get_the_post_thumbnail_url($loop->the_post()) ) {
                                echo get_the_post_thumbnail_url($loop->the_post(),'thumbnail');
                            } else {
                                echo $theme_uri. '/img/qut-logo.svg';
                            }
                            ?>
                                    );">
                                <figcaption style="position: absolute; display: table; height: 35%; width: 100%; background: rgba(0,0,0,0.6); color: white; font-size: 1rem; text-align: center; left: 0; bottom: 0; ">
                                    <div class="site-title" style="display: table-cell; vertical-align: middle; width: 100%; padding: 0px 10px;">
                                        <p style="margin: 0"><?= the_title(); ?></p>
                                    </div>
                                </figcaption>
                            </figure>
                        </a>
                    </div>
        <?php endwhile; ?>
        </div>

    <?php $output = ob_get_clean();
        return $output;
        \wp_reset_query();

    } else if ( ($atts['layout'] == 'list') ) {
        //ob_start(); ?>
        <ul style="margin-top:20px">
            <?php while ($loop->have_posts()) : $loop->the_post(); ?>
            <li><a href="<?php the_permalink(); ?>" title="<?php the_title_attribute(); ?>"><?php the_title(); ?></a></li>
            <?php endwhile; ?>
        </ul>
    <?php }

    \wp_reset_query();
    //ob_end_clean();
 }