Add `regex.find_offsets`, `regex.find`, improve `regex.match` (#1232)

`regex.match` now behaves like `string.match`.
This required changes in the `tokenizer` and in the `detectindent` 
plugin.

data/core/regex.lua

@@ -2,14 +2,85 @@
 -- pattern:gsub(string).
 regex.__index = function(table, key) return regex[key]; end
 
-regex.match = function(pattern_string, string, offset, options)
+---Looks for the first match of `pattern` in the string `str`.
-  local pattern = type(pattern_string) == "table" and
+---If it finds a match, it returns the indices of `str` where this occurrence
-    pattern_string or regex.compile(pattern_string)
+---starts and ends; otherwise, it returns `nil`.
-  local res = { regex.cmatch(pattern, string, offset or 1, options or 0) }
+---If the pattern has captures, the captured start and end indexes are returned,
-  res[2] = res[2] and res[2] - 1
+---after the two initial ones.
+---
+---@param pattern string|table The regex pattern to use, either as a simple string or precompiled.
+---@param str string The string to search for valid matches.
+---@param offset? integer The position on the subject to start searching.
+---@param options? integer A bit field of matching options, eg: regex.NOTBOL | regex.NOTEMPTY
+---
+---@return integer? start Offset where the first match was found; `nil` if no match.
+---@return integer? end Offset where the first match ends; `nil` if no match.
+---@return integer? ... #Captured matches offsets.
+regex.find_offsets = function(pattern, str, offset, options)
+  if type(pattern) ~= "table" then
+    pattern = regex.compile(pattern)
+  end
+  local res = { regex.cmatch(pattern, str, offset or 1, options or 0) }
+  -- Reduce every end delimiter by 1
+  for i = 2,#res,2 do
+    res[i] = res[i] - 1
+  end
   return table.unpack(res)
 end
 
+---Behaves like `string.match`.
+---Looks for the first match of `pattern` in the string `str`.
+---If it finds a match, it returns the matched string; otherwise, it returns `nil`.
+---If the pattern has captures, only the captured strings are returned.
+---If a capture is empty, its offset is returned instead.
+---
+---@param pattern string|table The regex pattern to use, either as a simple string or precompiled.
+---@param str string The string to search for valid matches.
+---@param offset? integer The position on the subject to start searching.
+---@param options? integer A bit field of matching options, eg: regex.NOTBOL | regex.NOTEMPTY
+---
+---@return (string|integer)? ... #List of captured matches; the entire match if no matches were specified; if the match is empty, its offset is returned instead.
+regex.match = function(pattern, str, offset, options)
+  local res = { regex.find(pattern, str, offset, options) }
+  if #res == 0 then return end
+  -- If available, only return captures
+  if #res > 2 then return table.unpack(res, 3) end
+  return string.sub(str, res[1], res[2])
+end
+
+---Behaves like `string.find`.
+---Looks for the first match of `pattern` in the string `str`.
+---If it finds a match, it returns the indices of `str` where this occurrence
+---starts and ends; otherwise, it returns `nil`.
+---If the pattern has captures, the captured strings are returned,
+---after the two indexes ones.
+---If a capture is empty, its offset is returned instead.
+---
+---@param pattern string|table The regex pattern to use, either as a simple string or precompiled.
+---@param str string The string to search for valid matches.
+---@param offset? integer The position on the subject to start searching.
+---@param options? integer A bit field of matching options, eg: regex.NOTBOL | regex.NOTEMPTY
+---
+---@return integer? start Offset where the first match was found; `nil` if no match.
+---@return integer? end Offset where the first match ends; `nil` if no match.
+---@return (string|integer)? ... #List of captured matches; if the match is empty, its offset is returned instead.
+regex.find = function(pattern, str, offset, options)
+  local res = { regex.find_offsets(pattern, str, offset, options) }
+  local out = { }
+  if #res == 0 then return end
+  out[1] = res[1]
+  out[2] = res[2]
+  for i = 3,#res,2 do
+    if res[i] > res[i+1] then
+      -- Like in string.find, if the group has size 0, return the index
+      table.insert(out, res[i])
+    else
+      table.insert(out, string.sub(str, res[i], res[i+1]))
+    end
+  end
+  return table.unpack(out)
+end
+
 -- Will iterate back through any UTF-8 bytes so that we don't replace bits
 -- mid character.
 local function previous_character(str, index)

data/core/tokenizer.lua

@@ -214,23 +214,12 @@ function tokenizer.tokenize(incoming_syntax, text, state)
         return
       end
       res = p.pattern and { text:ufind((at_start or p.whole_line[p_idx]) and "^" .. code or code, next) }
-        or { regex.match(code, text, text:ucharpos(next), (at_start or p.whole_line[p_idx]) and regex.ANCHORED or 0) }
+        or { regex.find(code, text, text:ucharpos(next), (at_start or p.whole_line[p_idx]) and regex.ANCHORED or 0) }
       if p.regex and #res > 0 then -- set correct utf8 len for regex result
-        local char_pos_1 = string.ulen(text:sub(1, res[1]))
+        local char_pos_1 = res[1] > next and string.ulen(text:sub(1, res[1])) or next
-        local char_pos_2 = char_pos_1 + string.ulen(text:sub(res[1], res[2])) - 1
+        local char_pos_2 = string.ulen(text:sub(1, res[2]))
-        -- `regex.match` returns group results as a series of `begin, end`
+        for i=3,#res do
-        -- we only want `begin`s
+          res[i] = string.ulen(text:sub(1, res[i] - 1)) + 1
-        if #res >= 3 then
-          res[3] = char_pos_1 + string.ulen(text:sub(res[1], res[3])) - 1
-        end
-        for i=1,(#res-3) do
-          local curr = i + 3
-          local from = i * 2 + 3
-          if from < #res then
-            res[curr] = char_pos_1 + string.ulen(text:sub(res[1], res[from])) - 1
-          else
-            res[curr] = nil
-          end
         end
         res[1] = char_pos_1
         res[2] = char_pos_2

data/plugins/detectindent.lua

@@ -190,11 +190,11 @@ local function get_non_empty_lines(syntax, lines)
                 end
               else
                 if comment[3] then
-                  local start, ending = regex.match(
+                  local start, ending = regex.find_offsets(
                     comment[2], line, 1, regex.ANCHORED
                   )
                   if start then
-                    if not regex.match(
+                    if not regex.find_offsets(
                         comment[3], line, ending+1, regex.ANCHORED
                       )
                     then
@@ -204,7 +204,7 @@ local function get_non_empty_lines(syntax, lines)
                     end
                     break
                   end
-                elseif regex.match(comment[2], line, 1, regex.ANCHORED) then
+                elseif regex.find_offsets(comment[2], line, 1, regex.ANCHORED) then
                   is_comment = true
                   break
                 end
@@ -214,7 +214,7 @@ local function get_non_empty_lines(syntax, lines)
             is_comment = true
             inside_comment = false
             end_pattern = nil
-          elseif end_regex and regex.match(end_regex, line) then
+          elseif end_regex and regex.find_offsets(end_regex, line) then
             is_comment = true
             inside_comment = false
             end_regex = nil

docs/api/regex.lua

@@ -41,8 +41,8 @@ regex.NOTEMPTY_ATSTART = 0x00000008
 ---@param pattern string
 ---@param options? regex.modifiers A string of one or more pattern modifiers.
 ---
----@return regex|string regex Ready to use regular expression object or error
+---@return regex|nil regex Ready to use regular expression object or nil on error.
----message if compiling the pattern failed.
+---@return string? error The error message if compiling the pattern failed.
 function regex.compile(pattern, options) end
 
 ---
@@ -53,7 +53,7 @@ function regex.compile(pattern, options) end
 ---@param options? integer A bit field of matching options, eg:
 ---regex.NOTBOL | regex.NOTEMPTY
 ---
----@return table<integer, integer> list List of offsets where a match was found.
+---@return integer ... list List of offsets where a match was found.
 function regex:cmatch(subject, offset, options) end