Merge pull request #8153 from dotnet/v3

🎉🎉🎉

docs/specs/escape.yml

@@ -13,6 +13,7 @@ repos:
       'a b.md':
       a&b.md:
       a&b.md:
+      a – b c/ab.md:
       a++.png:
 outputs:
   links.json: |
@@ -30,6 +31,7 @@ outputs:
   .publish.json: |
     {
       "files": [
+        { "url": "/a – b c/ab", "path": "a – b c/ab.json" },
         { "url": "/a b", "path": "a b.json" },
         { "url": "/a&b", "path": "a&b.json" },
         { "url": "/a&b", "path": "a&b.json" },
@@ -39,28 +41,32 @@ outputs:
     }
   'a b.json': |
     {
-      "canonical_url": "https://docs.com/en-us/a b",
-      "gitcommit": "https://github.com/escape/markdown/blob/c4c07cb69d114fa9ba16e55763d47f2344083dad/a b.md",
+      "canonical_url": "https://docs.com/en-us/a%20b",
+      "gitcommit": "https://github.com/escape/markdown/blob/03cb6e2fa19c69303dd589df63df0c40448b241b/a b.md",
       "content_git_url": "https://github.com/escape/markdown/blob/main/a b.md",
       "original_content_git_url": "https://github.com/escape/markdown/blob/main/a b.md",
       "original_content_git_url_template": "{repo}/blob/{branch}/a b.md"
     }
   a&b.json: |
     {
       "canonical_url": "https://docs.com/en-us/a&b",
-      "gitcommit": "https://github.com/escape/markdown/blob/c4c07cb69d114fa9ba16e55763d47f2344083dad/a&b.md",
+      "gitcommit": "https://github.com/escape/markdown/blob/03cb6e2fa19c69303dd589df63df0c40448b241b/a&b.md",
       "content_git_url": "https://github.com/escape/markdown/blob/main/a&b.md",
       "original_content_git_url": "https://github.com/escape/markdown/blob/main/a&b.md",
       "original_content_git_url_template": "{repo}/blob/{branch}/a&b.md"
     }
   a&b.json: |
     {
       "canonical_url": "https://docs.com/en-us/a&b",
-      "gitcommit": "https://github.com/escape/markdown/blob/c4c07cb69d114fa9ba16e55763d47f2344083dad/a&b.md",
+      "gitcommit": "https://github.com/escape/markdown/blob/03cb6e2fa19c69303dd589df63df0c40448b241b/a&b.md",
       "content_git_url": "https://github.com/escape/markdown/blob/main/a&b.md",
       "original_content_git_url": "https://github.com/escape/markdown/blob/main/a&b.md",
       "original_content_git_url_template": "{repo}/blob/{branch}/a&b.md"
     }
+  a – b c/ab.json: | 
+    {
+      "canonical_url": "https://docs.com/en-us/a%20%e2%80%93%20b%20c/ab"
+    }
   a++.png:
   .errors.log: |
     { "message_severity":"warning", "code":"file-not-found", "file":"links.md", "line":5 }
@@ -184,3 +190,40 @@ outputs:
     {
       "conceptual": "<p>the <span class=\"no-loc xref\">ServiceFuture&lt;String&gt;</span> object\n<span class=\"xref\">Observable&lt;Void&gt;</span></p>\n",
     }
+---
+# Canonical url for ugly url
+inputs:
+  docfx.yml: |
+    urlType: Ugly
+  a/index.md: |
+  a/main.md: |
+outputs:
+  a/index.json: |
+    {
+      "canonical_url": "https://docs.com/en-us/a/index.html",
+    }
+  a/main.json: |
+    {
+      "canonical_url": "https://docs.com/en-us/a/main.html",
+    }
+---
+# Escape Reserved Characters and Do not escape Unreserved Characters in Canonical url 
+inputs:
+  docfx.yml: 
+  a/reserved-#[]@.md: |
+  a/reserved2-!$&'()+,;=.md: |
+  a/unreserved-._~%.md: |
+outputs:
+  a/reserved-#[]@.json : | 
+    {
+      "canonical_url": "https://docs.com/en-us/a/reserved-%23%5b%5d@",
+    }
+  a/reserved2-!$&'()+,;=.json: |
+    {
+      "canonical_url": "https://docs.com/en-us/a/reserved2-!$&'()+,;=",
+    }
+  a/unreserved-._~%.json: |
+    {
+      "canonical_url": "https://docs.com/en-us/a/unreserved-._~%25",
+    }
+---

docs/specs/moniker.yml

@@ -2217,4 +2217,175 @@ repos:
 outputs:
   10b6c9f144d519f0f8a48bc02c54ca34/a.json: |
     {"conceptual": "<div data-moniker=\"tfs-2013\">\n<p>moniker: tfs-2013</p>\n</div>\n"}
-
+---
+# Append latest moniker to canonical url if configured
+inputs:
+  docfx.yml: |
+    isCanonicalUrlWithMoniker: true
+    monikerRange:
+      'docs/v1/**': '< netcore-2.0'
+      'docs/v2/**': '>= netcore-2.0'
+    hostName: docs.com
+    basePath: /docs
+    routes:
+      docs/: .
+      docs/v1/: .
+      docs/v2/: .
+    monikerDefinition:
+      monikers:
+      - { moniker_name: netcore-1.0, product_name: .NET Core }
+      - { moniker_name: netcore-1.1, product_name: .NET Core }
+      - { moniker_name: netcore-2.0, product_name: .NET Core }
+      - { moniker_name: netcore-2.1, product_name: .NET Core }
+  docs/v1/a.md: |
+    Moniker: netcore-1.0, netcore-1.1
+  docs/v2/b.md: |
+    Moniker: netcore-2.0, netcore-2.1
+  docs/v2/c.yml: |
+    ### YamlMime:TestData
+    metadata:
+      monikers: 
+      - netcore-2.0
+      - netcore-2.1
+  _themes/ContentTemplate/schemas/TestData.schema.json: |
+    {
+      "properties": {
+        "metadata": {
+          "properties": {
+            "monikers": { "type": "array" }
+          }
+        }
+      }
+    }  
+  _themes/ContentTemplate/TestData.html.primary.tmpl:
+outputs:
+  docs/17b9fe681514513cbf7d5c90e32f107a/a.json: |
+   {
+      "canonical_url": "https://docs.com/en-us/docs/a?view=netcore-1.1"
+   }
+  docs/ed8f7746ec932ae7c9f595c1f2c97d5a/b.json: |
+    {
+       "canonical_url": "https://docs.com/en-us/docs/b?view=netcore-2.1"
+    }
+  docs/ed8f7746ec932ae7c9f595c1f2c97d5a/c.json: |
+    {
+      "metadata": {
+        "canonical_url": "https://docs.com/en-us/docs/c?view=netcore-2.1"
+      }
+    }
+  .publish.json: |
+    {
+      "files":[
+        {
+          "url":"/docs/a",
+          "moniker_group":"17b9fe681514513cbf7d5c90e32f107a", 
+          "config_moniker_range": "< netcore-2.0",
+          "canonical_url": "https://docs.com/en-us/docs/a?view=netcore-1.1"
+        },
+        {
+          "url":"/docs/b",
+          "moniker_group":"ed8f7746ec932ae7c9f595c1f2c97d5a", 
+          "config_moniker_range": ">= netcore-2.0",
+          "canonical_url": "https://docs.com/en-us/docs/b?view=netcore-2.1"
+        },
+        {
+          "url":"/docs/c",
+          "moniker_group":"ed8f7746ec932ae7c9f595c1f2c97d5a", 
+          "config_moniker_range": ">= netcore-2.0",
+          "canonical_url": "https://docs.com/en-us/docs/c?view=netcore-2.1"
+        }
+      ],
+      "moniker_groups": {
+        "ed8f7746ec932ae7c9f595c1f2c97d5a": ["netcore-2.0","netcore-2.1"],
+        "17b9fe681514513cbf7d5c90e32f107a": ["netcore-1.0","netcore-1.1"]
+      }
+    }
+---
+# Overwrite canonical url for page json output
+inputs:
+  docfx.yml: |
+    outputType: pageJson
+    isCanonicalUrlWithMoniker: true
+    monikerRange:
+      'docs/v1/**': '< netcore-2.0'
+      'docs/v2/**': '>= netcore-2.0'
+    hostName: docs.com
+    basePath: /docs
+    routes:
+      docs/: .
+      docs/v1/: .
+      docs/v2/: .
+    monikerDefinition:
+      monikers:
+      - { moniker_name: netcore-1.0, product_name: .NET Core }
+      - { moniker_name: netcore-1.1, product_name: .NET Core }
+      - { moniker_name: netcore-2.0, product_name: .NET Core }
+      - { moniker_name: netcore-2.1, product_name: .NET Core }
+  docs/v1/a.md: |
+    Moniker: netcore-1.0, netcore-1.1
+  docs/v2/c.yml: |
+    ### YamlMime:TestData
+    metadata:
+      monikers: 
+      - netcore-2.0
+      - netcore-2.1
+  _themes/ContentTemplate/schemas/TestData.schema.json: |
+    {
+      "properties": {
+        "metadata": {
+          "properties": {
+            "monikers": { "type": "array" }
+          }
+        }
+      }
+    }  
+  _themes/ContentTemplate/TestData.html.primary.tmpl:
+outputs:
+  docs/17b9fe681514513cbf7d5c90e32f107a/a.raw.page.json: | 
+    {
+      "rawMetadata": {
+          "canonical_url": "https://docs.com/en-us/docs/a?view=netcore-1.1",
+          "_op_canonicalUrl": "https://docs.com/en-us/docs/a?view=netcore-1.1",
+      }
+    }
+  docs/17b9fe681514513cbf7d5c90e32f107a/a.mta.json: |
+    {
+      "canonical_url": "https://docs.com/en-us/docs/a?view=netcore-1.1",
+    }
+  docs/ed8f7746ec932ae7c9f595c1f2c97d5a/c.raw.page.json: | 
+    {
+      "rawMetadata": {
+          "metadata": {
+              "canonical_url": "https://docs.com/en-us/docs/c?view=netcore-2.1",
+            },
+          "_op_canonicalUrl": "https://docs.com/en-us/docs/c?view=netcore-2.1"
+      }
+    }
+  docs/ed8f7746ec932ae7c9f595c1f2c97d5a/c.mta.json: |
+    {
+      "metadata": {
+        "canonical_url": "https://docs.com/en-us/docs/c?view=netcore-2.1",
+      }
+    }
+  .publish.json: |
+    {
+      "files":[
+        {
+          "url":"/docs/a",
+          "moniker_group":"17b9fe681514513cbf7d5c90e32f107a", 
+          "config_moniker_range": "< netcore-2.0",
+          "canonical_url": "https://docs.com/en-us/docs/a?view=netcore-1.1"
+        },
+        {
+          "url":"/docs/c",
+          "moniker_group":"ed8f7746ec932ae7c9f595c1f2c97d5a", 
+          "config_moniker_range": ">= netcore-2.0",
+          "canonical_url": "https://docs.com/en-us/docs/c?view=netcore-2.1"
+        }
+      ],
+      "moniker_groups": {
+        "ed8f7746ec932ae7c9f595c1f2c97d5a": ["netcore-2.0","netcore-2.1"],
+        "17b9fe681514513cbf7d5c90e32f107a": ["netcore-1.0","netcore-1.1"]
+      }
+    }
+---

src/Microsoft.Docs.MarkdigExtensions/CodeSnippet/HtmlCodeSnippetRenderer.cs

@@ -468,10 +468,7 @@ public static bool TryGetLineRanges(string query, out List<CodeRange> codeRanges
                 return false;
             }
 
-            if (codeRanges == null)
-            {
-                codeRanges = new();
-            }
+            codeRanges ??= new();
 
             codeRanges.Add(codeRange);
         }

src/docfx/build/DocsetBuilder.cs

@@ -161,7 +161,7 @@ public void Build(string[]? files)
             var publishModelBuilder = new PublishModelBuilder(_config, _errors, _monikerProvider, _buildOptions, _sourceMap, _documentProvider, _contributionProvider);
             var resourceBuilder = new ResourceBuilder(_input, _documentProvider, _config, output, publishModelBuilder);
             var learnHierarchyBuilder = new LearnHierarchyBuilder(_contentValidator);
-            var pageBuilder = new PageBuilder(_config, _buildOptions, _input, output, _documentProvider, _metadataProvider, _monikerProvider, _templateEngine, _tocMap, _linkResolver, _xrefResolver, _contributionProvider, _bookmarkValidator, publishModelBuilder, _contentValidator, _metadataValidator, _markdownEngine, _redirectionProvider, _jsonSchemaTransformer, learnHierarchyBuilder);
+            var pageBuilder = new PageBuilder(_config, _buildOptions, _input, output, _documentProvider, _metadataProvider, _monikerProvider, _publishUrlMap, _templateEngine, _tocMap, _linkResolver, _xrefResolver, _contributionProvider, _bookmarkValidator, publishModelBuilder, _contentValidator, _metadataValidator, _markdownEngine, _redirectionProvider, _jsonSchemaTransformer, learnHierarchyBuilder);
             var tocBuilder = new TocBuilder(_config, _tocLoader, _contentValidator, _metadataProvider, _metadataValidator, _documentProvider, _monikerProvider, publishModelBuilder, _templateEngine, output);
             var redirectionBuilder = new RedirectionBuilder(publishModelBuilder, _redirectionProvider, _documentProvider);
 

src/docfx/build/document/DocumentProvider.cs

@@ -256,20 +256,22 @@ private static string PathToRelativeUrl(string path, ContentType contentType, Ur
             }
             if (urlType == UrlType.Docs && contentType != ContentType.Toc)
             {
+                // remove extension
                 var i = url.LastIndexOf('.');
                 return i >= 0 ? url[..i] : url;
             }
         }
         return url;
     }
 
-    /// <summary>
-    /// In docs, canonical URL is later overwritten by template JINT code.
-    /// TODO: need to handle the logic difference when template code is removed.
-    /// </summary>
-    private string GetCanonicalUrl(string siteUrl)
+    private string GetCanonicalUrl(string? siteUrl)
     {
-        return $"https://{_config.HostName}/{_buildOptions.Locale}{siteUrl}";
+        if (siteUrl == null)
+        {
+            return "";
+        }
+
+        return $"https://{_config.HostName}/{_buildOptions.Locale}{UrlUtility.EscapeUrlPath(siteUrl).ToLowerInvariant()}";
     }
 
     private PathString ApplyRoutes(PathString path)

src/docfx/build/link/LinkResolver.cs

@@ -235,11 +235,7 @@ public LinkResolver(
                 // which needs to be removed once the user input is correct
                 if (_templateEngine != null && JsonSchemaProvider.IsLandingData(_documentProvider.GetMime(inclusionRoot)))
                 {
-                    if (file is null)
-                    {
-                        // try to resolve with .md for landing page
-                        file = TryResolveRelativePath(referencingFile, $"{path}.md", lookupGitCommits, contentFallback);
-                    }
+                    file ??= TryResolveRelativePath(referencingFile, $"{path}.md", lookupGitCommits, contentFallback);
 
                     // Do not report error for landing page
                     return (null, file, query, fragment, linkType);

src/docfx/build/page/PageBuilder.cs

@@ -16,6 +16,7 @@ internal class PageBuilder
     private readonly DocumentProvider _documentProvider;
     private readonly MetadataProvider _metadataProvider;
     private readonly MonikerProvider _monikerProvider;
+    private readonly PublishUrlMap _publishUrlMap;
     private readonly TemplateEngine _templateEngine;
     private readonly TocMap _tocMap;
     private readonly LinkResolver _linkResolver;
@@ -38,6 +39,7 @@ public PageBuilder(
         DocumentProvider documentProvider,
         MetadataProvider metadataProvider,
         MonikerProvider monikerProvider,
+        PublishUrlMap publishUrlMap,
         TemplateEngine templateEngine,
         TocMap tocMap,
         LinkResolver linkResolver,
@@ -59,6 +61,7 @@ public PageBuilder(
         _documentProvider = documentProvider;
         _metadataProvider = metadataProvider;
         _monikerProvider = monikerProvider;
+        _publishUrlMap = publishUrlMap;
         _templateEngine = templateEngine;
         _tocMap = tocMap;
         _linkResolver = linkResolver;
@@ -232,7 +235,9 @@ private SystemMetadata CreateSystemMetadata(ErrorBuilder errors, FilePath file,
             _contributionProvider.GetContributionInfo(errors, file, userMetadata.Author);
 
         systemMetadata.Locale = _buildOptions.Locale;
-        systemMetadata.CanonicalUrl = userMetadata.PageType != "profile" ? _documentProvider.GetCanonicalUrl(file) : null;
+
+        systemMetadata.CanonicalUrl = GetCanonicalUrlWithMonikerIfNecessary(userMetadata.PageType, file);
+
         systemMetadata.Path = _documentProvider.GetSitePath(file);
         systemMetadata.Rel = PathUtility.GetRelativePathToRoot(systemMetadata.Path);
         systemMetadata.CanonicalUrlPrefix = UrlUtility.Combine($"https://{_config.HostName}", systemMetadata.Locale, _config.BasePath) + "/";
@@ -261,6 +266,27 @@ private SystemMetadata CreateSystemMetadata(ErrorBuilder errors, FilePath file,
         return systemMetadata;
     }
 
+    private string? GetCanonicalUrlWithMonikerIfNecessary(string? pageType, FilePath file)
+    {
+        if (pageType == "profile")
+        {
+            return null;
+        }
+
+        var canonicalUrl = _documentProvider.GetCanonicalUrl(file);
+
+        if (_config.IsCanonicalUrlWithMoniker)
+        {
+            var canonicalVersion = _publishUrlMap.GetCanonicalVersion(file);
+            if (!string.IsNullOrEmpty(canonicalVersion))
+            {
+                return $"{canonicalUrl}?view={UrlUtility.EscapeUrlQueryOrFragment(canonicalVersion)}";
+            }
+        }
+
+        return canonicalUrl;
+    }
+
     private JObject LoadMarkdown(ErrorBuilder errors, FilePath file)
     {
         var content = _input.ReadString(file);