Go to /Glimpse.axd
and uncheck all the Glimpse addons/tabs. It will give you the XML config needed to copy & paste into your web.config. In my case, it was Routes, probably due to extensive use of attribute routing. Others have mentioned Cache and Metadata, so start by ignoring all, then comment out a few at a time until you find what slows down Glimpse.
Note that the below config may not work for you, if you have different addons or versions installed.
<glimpse defaultRuntimePolicy="On" endpointBaseUri="~/Glimpse.axd">
<tabs>
<ignoredTypes>
<!-- <add type="Glimpse.Ado.Tab.SQL, Glimpse.Ado" />-->
<!-- <add type="Glimpse.AspNet.Tab.Cache, Glimpse.AspNet" />-->
<!-- <add type="Glimpse.AspNet.Tab.Configuration, Glimpse.AspNet" />-->
<!-- <add type="Glimpse.AspNet.Tab.Environment, Glimpse.AspNet" />-->
<!-- <add type="Glimpse.AspNet.Tab.Request, Glimpse.AspNet" />-->
<add type="Glimpse.AspNet.Tab.Routes, Glimpse.AspNet" />
<!-- <add type="Glimpse.AspNet.Tab.Server, Glimpse.AspNet" />-->
<!-- <add type="Glimpse.AspNet.Tab.Session, Glimpse.AspNet" />-->
<!-- <add type="Glimpse.Core.Tab.Timeline, Glimpse.Core" />-->
<!-- <add type="Glimpse.Core.Tab.Trace, Glimpse.Core" />-->
<!-- <add type="Glimpse.Mvc.Tab.Execution, Glimpse.Mvc5" />-->
<!-- <add type="Glimpse.Mvc.Tab.Metadata, Glimpse.Mvc5" />-->
<!-- <add type="Glimpse.Mvc.Tab.Views, Glimpse.Mvc5" />-->
<!-- <add type="Glimpse.NLog.NLogTab, Glimpse.NLog" /> -->
</ignoredTypes>
</tabs>
<runtimePolicies>
</runtimePolicies>
</glimpse>