Ended up copying some decompiled Episerver code and disabling  .FilterOnCurrentSite() in the .PublicSearchFilter()

This should be placed in an InitializableModule.

// Code from CmsUnifiedSearchSetUp.cs in Episerver.Find.Cms DLL 13.0.5
var cmsUnifiedSearchSetup = new EPiServer.Find.Cms.CmsUnifiedSearchSetUp();

SearchClient.Instance.Conventions.UnifiedSearchRegistry.Add<PageData>()
    .PublicSearchFilter((Func<IClient, ISearchContext, Filter>)((c, ctx) => (Filter)c.BuildFilter<IContentData>()
        .FilterForVisitor<IContentData>(this.GetLanguage(ctx)
        ).ExcludeContainerPages<IContentData>()
        .ExcludeContentFolders<IContentData>()
        //.FilterOnCurrentSite<IContentData>()
        ))
    .CustomizeIndexProjection(new Func<IndexProjectionBuilder, IndexProjectionBuilder>(cmsUnifiedSearchSetup.CustomizeIndexProjectionForPageData))
    .CustomizeProjection(new Func<HitProjectionBuilder, HitProjectionBuilder>(cmsUnifiedSearchSetup.CustomizeProjectionForPageData));

SearchClient.Instance.Conventions.UnifiedSearchRegistry.Add<MediaData>()
    .PublicSearchFilter((Func<IClient, ISearchContext, Filter>)((c, ctx) => (Filter)c.BuildFilter<IContentData>()
        .FilterForVisitor<IContentData>(this.GetLanguage(ctx))
        .ExcludeContentFolders<IContentData>()
        //.FilterOnCurrentSite<IContentData>()
        ))
    .CustomizeIndexProjection(new Func<IndexProjectionBuilder, IndexProjectionBuilder>(cmsUnifiedSearchSetup.CustomizeIndexProjectionForMediaData))
    .CustomizeProjection(new Func<HitProjectionBuilder, HitProjectionBuilder>(cmsUnifiedSearchSetup.CustomizeProjectionForMediaData));

// Code from CmsUnifiedSearchSetUp.cs in Episerver.Find.Cms DLL 13.0.5
private string GetLanguage(ISearchContext context)
{
    if (context.Payload.ContainsKey("Culture") && context.Payload["Culture"].IsNotNull())
        return ((CultureInfo)context.Payload["Culture"]).Name;
    if (context.ContentLanguage == null || context.ContentLanguage == Language.None)
        return Languages.AllLanguagesSuffix;
    return context.ContentLanguage.FieldSuffix;
}