Link syntax require to escape every square bracket

* lisp/ol.el (org-link-make-regexps): Update regexp to forbid any
un-escaped square bracket in the URI.
(org-link-escape):
(org-link-unescape):
* testing/lisp/test-ol.el (test-ol/escape):
(test-ol/unescape):
(test-ol/store-link):
* testing/lisp/test-org.el (test-org/custom-id):
(test-org/fuzzy-links):
* testing/lisp/test-ox.el (test-org-export/resolve-fuzzy-link): Adapt
to new syntax.
* doc/org-manual.org (Link Format): Update documentation.

The new syntax allowed un-escaped opening square brackets in the URI
part of bracket links. Unfortunately, it led to bug as described here:

  <https://lists.gnu.org/archive/html/emacs-orgmode/2019-12/msg00312.html>

Now, we require to escape every square bracket in the URI.

doc/org-manual.org

@@ -2940,23 +2940,21 @@ or alternatively
 
 #+cindex: escape syntax, for links
 #+cindex: backslashes, in links
-Some =\= and =]= characters in the {{{var(LINK)}}} part need to be
-"escaped", i.e., preceded by another =\= character.  More
-specifically, the following character categories, and only them, must
-be escaped, in order:
+Some =\=, =[= and =]= characters in the {{{var(LINK)}}} part need to
+be "escaped", i.e., preceded by another =\= character.  More
+specifically, the following characters, and only them, must be
+escaped:
 
-1. all consecutive =\= characters at the end of the link,
-2. any =]= character at the very end of the link,
-3. all consecutive =\= characters preceding =][= or =]]= patterns,
-4. any =]= character followed by either =[= or =]=.
+1. all =[= and =]= characters,
+2. every =\= character preceding either =]= or =[=,
+3. every =\= character at the end of the link.
 
 #+findex: org-link-escape
-Org takes for granted that such links are correctly escaped.
-Functions inserting links (see [[*Handling Links]]) take care of this.
-You only need to bother about those rules when inserting directly, or
-yanking, a URI within square brackets.  When in doubt, you may use the
-function ~org-link-escape~, which turns a link string into its
-properly escaped form.
+Functions inserting links (see [[*Handling Links]]) properly escape
+ambiguous characters.  You only need to bother about the rules above
+when inserting directly, or yanking, a URI within square brackets.
+When in doubt, you may use the function ~org-link-escape~, which turns
+a link string into its escaped form.
 
 Once a link in the buffer is complete, with all brackets present, Org
 changes the display so that =DESCRIPTION= is displayed instead of

etc/ORG-NEWS

@@ -19,15 +19,11 @@ Org used to percent-encode sensitive characters in the URI part of the
 bracket links.
 
 Now, escaping mechanism uses the usual backslash character, according
-to the following rules, applied in order:
-
-1. All consecutive =\= characters at the end of the link must be
-   escaped;
-2. Any =]= character at the very end of the link must be escaped;
-3. All consecutive =\= characters preceding =][= or =]]= patterns must
-   be escaped;
-4. Any =]= character followed by either =[= or =]= must be escaped;
-5. Others =]= and =\= characters need not be escaped.
+to the following rules:
+
+1. All =[= and =]= characters in the URI must be escaped;
+2. Every =\= character preceding either =[= or =]= must be escaped;
+3. Every =\= character at the end of the URI must be escaped.
 
 When in doubt, use the function ~org-link-escape~ in order to turn
 a link string into its properly escaped form.

lisp/ol.el

@@ -716,12 +716,10 @@ This should be called after the variable `org-link-parameters' has changed."
 	  (rx (seq "[["
 		   ;; URI part: match group 1.
 		   (group
-		    ;; Allow an even number of backslashes right
-		    ;; before the closing bracket.
-		    (or (one-or-more "\\\\")
-			(and (*? anything)
-			     (not (any "\\"))
-			     (zero-or-more "\\\\"))))
+		    (one-or-more
+                     (or (not (any "[]\\"))
+			 (and "\\" (zero-or-more "\\\\") (any "[]"))
+			 (and (one-or-more "\\") (not (any "[]"))))))
 		   "]"
 		   ;; Description (optional): match group 2.
 		   (opt "[" (group (+? anything)) "]")
@@ -838,30 +836,21 @@ E.g. \"%C3%B6\" becomes the german o-Umlaut."
 
 (defun org-link-escape (link)
   "Backslash-escape sensitive characters in string LINK."
-  ;; Escape closing square brackets followed by another square bracket
-  ;; or at the end of the link.  Also escape final backslashes so that
-  ;; we do not escape inadvertently URI's closing bracket.
-  (with-temp-buffer
-    (insert link)
-    (insert (make-string (- (skip-chars-backward "\\\\"))
-			 ?\\))
-    (while (search-backward "\]" nil t)
-      (when (looking-at-p "\\]\\(?:[][]\\|\\'\\)")
-	(insert (make-string (1+ (- (skip-chars-backward "\\\\")))
-			     ?\\))))
-    (buffer-string)))
+  (replace-regexp-in-string
+   (rx (seq (group (zero-or-more "\\")) (group (or string-end (any "[]")))))
+   (lambda (m)
+     (concat (match-string 1 m)
+	     (match-string 1 m)
+	     (and (/= (match-beginning 2) (match-end 2)) "\\")))
+   link nil t 1))
 
 (defun org-link-unescape (link)
   "Remove escaping backslash characters from string LINK."
-  (with-temp-buffer
-    (save-excursion (insert link))
-    (while (re-search-forward "\\(\\\\+\\)\\]\\(?:[][]\\|\\'\\)" nil t)
-      (replace-match (make-string (/ (- (match-end 1) (match-beginning 1)) 2)
-				  ?\\)
-		     nil t nil 1))
-    (goto-char (point-max))
-    (delete-char (/ (- (skip-chars-backward "\\\\")) 2))
-    (buffer-string)))
+  (replace-regexp-in-string
+   (rx (group (one-or-more "\\")) (or string-end (any "[]")))
+   (lambda (_)
+     (concat (make-string (/ (- (match-end 1) (match-beginning 1)) 2) ?\\)))
+   link nil t 1))
 
 (defun org-link-make-string (link &optional description)
   "Make a bracket link, consisting of LINK and DESCRIPTION.

testing/lisp/test-ol.el

@@ -55,49 +55,48 @@
 
 (ert-deftest test-ol/escape ()
   "Test `org-link-escape' specifications."
-  ;; No-op when there is no backslash or closing square bracket.
-  (should (string= "foo[" (org-link-escape "foo[")))
-  ;; Escape closing square bracket at the end of the link.
-  (should (string= "[foo\\]" (org-link-escape "[foo]")))
-  ;; Escape closing square brackets followed by another square
-  ;; bracket.
-  (should (string= "foo\\][bar" (org-link-escape "foo][bar")))
-  (should (string= "foo\\]]bar" (org-link-escape "foo]]bar")))
-  ;; However, escaping closing square bracket at the end of the link
-  ;; has precedence over the previous rule.
-  (should (string= "foo]\\]" (org-link-escape "foo]]")))
+  ;; No-op when there is no backslash or square bracket.
+  (should (string= "foo" (org-link-escape "foo")))
+  ;; Escape square brackets at boundaries of the link.
+  (should (string= "\\[foo\\]" (org-link-escape "[foo]")))
+  ;; Escape square brackets followed by another square bracket.
+  (should (string= "foo\\]\\[bar" (org-link-escape "foo][bar")))
+  (should (string= "foo\\]\\]bar" (org-link-escape "foo]]bar")))
+  (should (string= "foo\\[\\[bar" (org-link-escape "foo[[bar")))
+  (should (string= "foo\\[\\]bar" (org-link-escape "foo[]bar")))
   ;; Escape backslashes at the end of the link.
   (should (string= "foo\\\\" (org-link-escape "foo\\")))
   ;; Escape backslashes that could be confused with escaping
   ;; characters.
   (should (string= "foo\\\\\\]" (org-link-escape "foo\\]")))
-  (should (string= "foo\\\\\\][" (org-link-escape "foo\\][")))
-  (should (string= "foo\\\\\\]]bar" (org-link-escape "foo\\]]bar")))
+  (should (string= "foo\\\\\\]\\[" (org-link-escape "foo\\][")))
+  (should (string= "foo\\\\\\]\\]bar" (org-link-escape "foo\\]]bar")))
   ;; Do not escape backslash characters when unnecessary.
   (should (string= "foo\\bar" (org-link-escape "foo\\bar")))
-  (should (string= "foo\\]bar" (org-link-escape "foo\\]bar")))
   ;; Pathological cases: consecutive closing square brackets.
-  (should (string= "[[[foo\\]]\\]" (org-link-escape "[[[foo]]]")))
-  (should (string= "[[[foo]\\]] bar" (org-link-escape "[[[foo]]] bar"))))
+  (should (string= "\\[\\[\\[foo\\]\\]\\]" (org-link-escape "[[[foo]]]")))
+  (should (string= "\\[\\[foo\\]\\] bar" (org-link-escape "[[foo]] bar"))))
 
 (ert-deftest test-ol/unescape ()
   "Test `org-link-unescape' specifications."
   ;; No-op if there is no backslash.
-  (should (string= "foo[" (org-link-unescape "foo[")))
+  (should (string= "foo" (org-link-unescape "foo")))
   ;; No-op if backslashes are not escaping backslashes.
   (should (string= "foo\\bar" (org-link-unescape "foo\\bar")))
-  (should (string= "foo\\]bar" (org-link-unescape "foo\\]bar")))
-  ;;
+  ;; Unescape backslashes before square brackets.
+  (should (string= "foo]bar" (org-link-unescape "foo\\]bar")))
   (should (string= "foo\\]" (org-link-unescape "foo\\\\\\]")))
   (should (string= "foo\\][" (org-link-unescape "foo\\\\\\][")))
-  (should (string= "foo\\]]bar" (org-link-unescape "foo\\\\\\]]bar")))
+  (should (string= "foo\\]]bar" (org-link-unescape "foo\\\\\\]\\]bar")))
+  (should (string= "foo\\[[bar" (org-link-unescape "foo\\\\\\[\\[bar")))
+  (should (string= "foo\\[]bar" (org-link-unescape "foo\\\\\\[\\]bar")))
   ;; Unescape backslashes at the end of the link.
   (should (string= "foo\\" (org-link-unescape "foo\\\\")))
-  ;; Unescape closing square bracket at the end of the link.
-  (should (string= "[foo]" (org-link-unescape "[foo\\]")))
+  ;; Unescape closing square bracket at boundaries of the link.
+  (should (string= "[foo]" (org-link-unescape "\\[foo\\]")))
   ;; Pathological cases: consecutive closing square brackets.
-  (should (string= "[[[foo]]]" (org-link-unescape "[[[foo\\]]\\]")))
-  (should (string= "[[[foo]]] bar" (org-link-unescape "[[[foo]\\]] bar"))))
+  (should (string= "[[[foo]]]" (org-link-unescape "\\[\\[\\[foo\\]\\]\\]")))
+  (should (string= "[[foo]] bar" (org-link-unescape "\\[\\[foo\\]\\] bar"))))
 
 (ert-deftest test-ol/make-string ()
   "Test `org-link-make-string' specifications."
@@ -204,11 +203,11 @@
   ;; Store file link to non-Org buffer, with context.
   (should
    (let ((org-stored-links nil)
-	 (org-context-in-file-links t))
+	 (org-link-context-for-files t))
      (org-test-with-temp-text-in-file "one\n<point>two"
        (fundamental-mode)
        (let ((file (buffer-file-name)))
-	 (equal (format "[[file:%s::one]]" file)
+	 (equal (format "[[file:%s::two]]" file)
 		(org-store-link nil))))))
   ;; Store file link to non-Org buffer, without context.
   (should
@@ -223,11 +222,11 @@
   ;; buffer.
   (should
    (let ((org-stored-links nil)
-	 (org-context-in-file-links nil))
+	 (org-link-context-for-files nil))
      (org-test-with-temp-text-in-file "one\n<point>two"
        (fundamental-mode)
        (let ((file (buffer-file-name)))
-	 (equal (format "[[file:%s::one]]" file)
+	 (equal (format "[[file:%s::two]]" file)
 		(org-store-link '(4)))))))
   ;; A C-u C-u does *not* reverse `org-context-in-file-links' in
   ;; non-Org buffer.

testing/lisp/test-org.el

@@ -2331,7 +2331,7 @@ SCHEDULED: <2014-03-04 tue.>"
   ;; Handle escape characters.
   (should
    (org-test-with-temp-text
-       "* H1\n:PROPERTIES:\n:CUSTOM_ID: [%]\n:END:\n* H2\n[[#[%\\]<point>]]"
+       "* H1\n:PROPERTIES:\n:CUSTOM_ID: [%]\n:END:\n* H2\n[[#\\[%\\]<point>]]"
      (org-open-at-point)
      (looking-at-p "\\* H1")))
   ;; Throw an error on false positives.
@@ -2427,7 +2427,7 @@ Foo Bar
      (looking-at "\\* TODO COMMENT Test")))
   ;; Correctly un-escape fuzzy links.
   (should
-   (org-test-with-temp-text "* [foo]\n[[*[foo\\]][With escaped characters]]"
+   (org-test-with-temp-text "* [foo]\n[[*\\[foo\\]][With escaped characters]]"
      (org-open-at-point)
      (bobp)))
   ;; Match search strings containing newline characters, including

testing/lisp/test-ox.el

@@ -3555,7 +3555,7 @@ Another text. (ref:text)
 	   (org-element-map tree 'link 'identity info t) info)))))
   ;; Handle escaped fuzzy links.
   (should
-   (org-test-with-parsed-data "* [foo]\n[[[foo\\]]]"
+   (org-test-with-parsed-data "* [foo]\n[[\\[foo\\]]]"
      (org-export-resolve-fuzzy-link
       (org-element-map tree 'link #'identity info t) info))))