feat: allow special characters to pass through slugification

dev-test/config.yml

@@ -7,6 +7,17 @@ publish_mode: editorial_workflow
 media_folder: assets/uploads
 
 collections: # A list of collections the CMS should be able to edit
+  - name: 'nested'
+    label: 'Nested Pages'
+    label_singular: 'Nested Page'
+    folder: '_posts'
+    slug: '{{fields.slug}}'
+    nested: {depth: 100}
+    create: true
+    fields: # The fields each document in this collection have
+      - { label: 'Title', name: 'title', widget: 'string', tagname: 'h1' }
+      - { label: 'Slug', name: 'slug', widget: 'string' }
+      - { label: 'Body', name: 'body', widget: 'markdown' }
   - name: 'posts' # Used in routes, ie.: /admin/collections/:slug/edit
     label: 'Posts' # Used in the UI
     label_singular: 'Post' # Used in the UI, ie: "New Post"

packages/netlify-cms-core/src/actions/config.ts

@@ -234,10 +234,6 @@ export function applyDefaults(originalConfig: CmsConfig) {
       config.slug.sanitize_replacement = '-';
     }
 
-    if (!('allowed_chars' in config.slug)) {
-      config.slug.allowed_chars = '';
-    }
-
     const i18n = config[I18N];
 
     if (i18n) {

packages/netlify-cms-core/src/lib/__tests__/urlHelper.spec.js

@@ -125,6 +125,13 @@ describe('sanitizeSlug', () => {
       'test_test',
     );
   });
+
+  it('preserves slashes if when requested', () => {
+    const input = '/this-is-a/nested/page';
+
+    expect(sanitizeSlug(input, slugConfig, false)).toEqual('this-is-a-nested-page');
+    expect(sanitizeSlug(input, slugConfig, true)).toEqual('this-is-a/nested/page');
+  });
 });
 
 describe('sanitizeChar', () => {

packages/netlify-cms-core/src/lib/formatters.ts

@@ -104,11 +104,19 @@ export function prepareSlug(slug: string) {
   );
 }
 
-export function getProcessSegment(slugConfig?: CmsSlug, ignoreValues?: string[]) {
+export function getProcessSegment(
+  slugConfig?: CmsSlug,
+  ignoreValues?: string[],
+  preserveSlash?: boolean,
+) {
   return (value: string) =>
     ignoreValues && ignoreValues.includes(value)
       ? value
-      : flow([value => String(value), prepareSlug, partialRight(sanitizeSlug, slugConfig)])(value);
+      : flow([
+          value => String(value),
+          prepareSlug,
+          partialRight(sanitizeSlug, slugConfig, preserveSlash),
+        ])(value);
 }
 
 export function slugFormatter(
@@ -185,10 +193,11 @@ export function previewUrlFormatter(
   fields = addFileTemplateFields(entry.get('path'), fields, collection.get('folder'));
   const dateFieldName = getDateField() || selectInferedField(collection, 'date');
   const date = parseDateFromEntry(entry as unknown as Map<string, unknown>, dateFieldName);
+  const preserveSlash = collection.has('nested');
 
   // Prepare and sanitize slug variables only, leave the rest of the
   // `preview_path` template as is.
-  const processSegment = getProcessSegment(slugConfig, [fields.get('dirname')]);
+  const processSegment = getProcessSegment(slugConfig, [fields.get('dirname')], preserveSlash);
   let compiledPath;
 
   try {

packages/netlify-cms-core/src/lib/urlHelper.ts

@@ -2,7 +2,7 @@ import url from 'url';
 import urlJoin from 'url-join';
 import diacritics from 'diacritics';
 import sanitizeFilename from 'sanitize-filename';
-import { isString, escapeRegExp, flow, partialRight } from 'lodash';
+import { isString, escapeRegExp, flow, partialRight, identity } from 'lodash';
 
 import type { CmsSlug } from '../types/redux';
 
@@ -48,7 +48,14 @@ function validIRIChar(char: string) {
   return uriChars.test(char) || ucsChars.test(char);
 }
 
-export function getCharReplacer(encoding: string, replacement: string, allowed: string) {
+export function getCharReplacer(
+  encoding: string,
+  options: {
+    replacement: NonNullable<CmsSlug['sanitize_replacement']>;
+    preserveSlash?: boolean;
+  },
+) {
+  const { replacement, preserveSlash } = options;
   let validChar: (char: string) => boolean;
 
   if (encoding === 'unicode') {
@@ -64,17 +71,24 @@ export function getCharReplacer(encoding: string, replacement: string, allowed:
     throw new Error('The replacement character(s) (options.replacement) is itself unsafe.');
   }
 
-  return (char: string) => {
-    return allowed.includes(char) || validChar(char) ? char : replacement;
+  return (char: string, i = 0, arr: string[] = [char]) => {
+    if (preserveSlash && char === '/' && i !== 0 && i !== arr.length - 1) {
+      return char;
+    }
+
+    return validChar(char) ? char : replacement;
   };
 }
 // `sanitizeURI` does not actually URI-encode the chars (that is the browser's and server's job), just removes the ones that are not allowed.
-export function sanitizeURI(str: string, options?: CmsSlug) {
-  const {
-    sanitize_replacement: replacement = '',
-    encoding = 'unicode',
-    allowed_chars = '',
-  } = options || {};
+export function sanitizeURI(
+  str: string,
+  options?: {
+    replacement: CmsSlug['sanitize_replacement'];
+    encoding: CmsSlug['encoding'];
+    preserveSlash?: boolean;
+  },
+) {
+  const { replacement = '', encoding = 'unicode', preserveSlash } = options || {};
 
   if (!isString(str)) {
     throw new Error('The input slug must be a string.');
@@ -85,29 +99,29 @@ export function sanitizeURI(str: string, options?: CmsSlug) {
 
   // `Array.from` must be used instead of `String.split` because
   //   `split` converts things like emojis into UTF-16 surrogate pairs.
-  return Array.from(str).map(getCharReplacer(encoding, replacement, allowed_chars)).join('');
+  return Array.from(str).map(getCharReplacer(encoding, { replacement, preserveSlash })).join('');
 }
 
 export function sanitizeChar(char: string, options?: CmsSlug) {
-  const {
-    encoding = 'unicode',
-    sanitize_replacement: replacement = '',
-    allowed_chars = '',
-  } = options || {};
-  return getCharReplacer(encoding, replacement, allowed_chars)(char);
+  const { encoding = 'unicode', sanitize_replacement: replacement = '' } = options || {};
+  return getCharReplacer(encoding, { replacement })(char);
 }
 
-export function sanitizeSlug(str: string, options?: CmsSlug) {
+export function sanitizeSlug(str: string, options?: CmsSlug, preserveSlash?: boolean) {
   if (!isString(str)) {
     throw new Error('The input slug must be a string.');
   }
 
-  const { clean_accents: stripDiacritics, sanitize_replacement: replacement } = options || {};
+  const {
+    encoding,
+    clean_accents: stripDiacritics,
+    sanitize_replacement: replacement,
+  } = options || {};
 
   const sanitizedSlug = flow([
     ...(stripDiacritics ? [diacritics.remove] : []),
-    partialRight(sanitizeURI, options),
-    partialRight(sanitizeFilename, { replacement }),
+    partialRight(sanitizeURI, { replacement, encoding, preserveSlash }),
+    preserveSlash ? identity : partialRight(sanitizeFilename, { replacement }),
   ])(str);
 
   // Remove any doubled or leading/trailing replacement characters (that were added in the sanitizers).

packages/netlify-cms-core/src/types/redux.ts

@@ -380,7 +380,6 @@ export interface CmsSlug {
   encoding?: CmsSlugEncoding;
   clean_accents?: boolean;
   sanitize_replacement?: string;
-  allowed_chars?: string;
 }
 
 export interface CmsLocalBackend {

website/yarn.lock

@@ -10252,9 +10252,9 @@ pretty-format@^25.5.0:
     react-is "^16.12.0"
 
 prismjs@^1.21.0:
-  version "1.27.0"
-  resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.27.0.tgz#bb6ee3138a0b438a3653dd4d6ce0cc6510a45057"
-  integrity sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA==
+  version "1.26.0"
+  resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.26.0.tgz#16881b594828bb6b45296083a8cbab46b0accd47"
+  integrity sha512-HUoH9C5Z3jKkl3UunCyiD5jwk0+Hz0fIgQ2nbwU2Oo/ceuTAQAg+pPVnfdt2TJWRVLcxKh9iuoYDUSc8clb5UQ==
 
 process-nextick-args@~2.0.0:
   version "2.0.1"