How can a TinyMCE modal return formatted/visual text?

(Revised answer)

You can use tinymce.DOM.encode() to convert all HTML tags to their entities, e.g. < for < and > for >:

var html = tinymce.DOM.encode(e.data.code);

Then to preserve trailing white-spaces:

html = html.replace(/(^ +| +$)/gm, function(match, p1){
  return p1.replace(/ /g, ' ');
});

And this to convert all line breaks to <br>:

html = html.replace(/(?:\r\n|\r|\n)/g, '<br>');

So your onSubmit would look like, which you can try here:

onSubmit: function(e) {
  ed.focus();

  ed.undoManager.transact(function() {
    // Encode all HTML tags to their entities.
    var html = tinymce.DOM.encode(e.data.code);

    // Then convert trailing whitespaces to ` `.
    html = html.replace(/(^ +| +$)/gm, function(match, p1){
      return p1.replace(/ /g, ' ');
    });

    // Finally, convert line breaks to `<br>`.
    html = html.replace(/(?:\r\n|\r|\n)/g, '<br>');

    // Make sure the format is "raw".
    ed.insertContent('<code>' + html + '</code> ');
  });

  ed.selection.setCursorLocation();
  ed.nodeChanged();
}

Credit to this SO answer for the line breaks conversion.