It is usually not that big. Were you able to debug to see what is happening?
I think you can override SitemapController > Index routing

You can override to see what's happening. Here is an example of our implementation, In our case we had two sites to take care of. Hence differentiating based on StartPage.ID. See more in Generate method.

[Route("sitemapindex.xml")]
public class MultisiteGetaSitemapIndexController : BaseController
{
    private readonly ISitemapRepository _sitemapRepository;

    public MultisiteGetaSitemapIndexController(ISitemapRepository sitemapRepository)
    {
        _sitemapRepository = sitemapRepository;
    }

    private static XNamespace SitemapXmlNamespace
    {
        get { return @"http://www.sitemaps.org/schemas/sitemap/0.9"; }
    }

    [Route("", Name = "Sitemap index")]
    [HttpGet]
    public new IActionResult Index()
    {
        // Call the Generate method when accessing sitemapindex.xml
        return Generate();
    }

    [HttpGet(nameof(Generate))]
    public IActionResult Generate()
    {
        var doc = new XDocument(new XDeclaration("1.0", "utf-8", null));

        var indexElement = new XElement(SitemapXmlNamespace + "sitemapindex");

        foreach (var sitemapData in _sitemapRepository.GetAllSitemapData()
                     .Where(x => x.RootPageId == ContentReference.StartPage.ID))
        {
            var sitemapElement = new XElement(
                SitemapXmlNamespace + "sitemap",
                new XElement(SitemapXmlNamespace + "loc", _sitemapRepository.GetSitemapUrl(sitemapData)));

            indexElement.Add(sitemapElement);
        }

        doc.Add(indexElement);

        byte[] sitemapIndexData;

        using (var ms = new MemoryStream())
        {
            using (var xtw = new XmlTextWriter(ms, Encoding.UTF8))
            {
                doc.Save(xtw);
                xtw.Flush();
                sitemapIndexData = ms.ToArray();
            }
        }

        return new FileContentResult(sitemapIndexData, "text/xml");
    }
}