add the ability to match EOF

Author: exaexa

Project.toml

@@ -1,7 +1,7 @@
 name = "PikaParser"
 uuid = "3bbf5609-3e7b-44cd-8549-7c69f321e792"
 authors = ["The developers of PikaParser.jl"]
-version = "0.4.0"
+version = "0.5.0"
 
 [deps]
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"

README.md

@@ -74,14 +74,13 @@ P.find_match_at!(p, :expr, 1)
 ```
 ...which returns an index in the match table (if found), such as `45`.
 
-You can have a look at the match: `p.matches[45]` should return:
-```julia
-PikaParser.Match(10, 1, 13, 2, [44])
-```
-where `10` is the renumbered rule ID for `:expr`, `1` is the starting position
-in the input, `13` is the length of the match (here, that is the whole input);
-`2` is the option index (in this case, it points to `:expr` option 2, which is
-`:minusexpr`), and 44 is the submatch of `:minusexpr`.
+You can have a look at the match: `p.matches[45]` should return: ```julia
+PikaParser.Match(10, 1, 13, 2, 52, 0, 41, 0) ``` where `10` is the renumbered
+rule ID for `:expr`, `1` is the starting position of the match in the input,
+`13` is the last position of the match (here, that means the whole input); `2`
+is the option index (in this case, it points to `:expr` option 2, which is
+`:minusexpr`). The rest of the `Match` structure is used for internal values
+that organize the match tree and submatches.
 
 ### Recovering parsed ASTs
 

src/clauses.jl

@@ -46,6 +46,7 @@ rechildren(x::Scan, t::DataType, v::Vector) = Scan{valtype(v),t}(x.match)
 rechildren(x::Token, t::DataType, v::Vector) = Token{valtype(v),t}(x.token)
 rechildren(x::Tokens, t::DataType, v::Vector) =
     Tokens{valtype(v),t,typeof(x.tokens)}(x.tokens)
+rechildren(x::EndOfInput, t::DataType, v::Vector) = EndOfInput{valtype(v),t}()
 rechildren(x::Epsilon, t::DataType, v::Vector) = Epsilon{valtype(v),t}()
 rechildren(x::Fail, t::DataType, v::Vector) = Fail{valtype(v),t}()
 rechildren(x::Seq, t::DataType, v::Vector) = Seq{valtype(v),t}(v)
@@ -97,7 +98,7 @@ better_match_than(::Clause, new::Match, old::Match) = new.last > old.last
 
 
 can_match_epsilon(x::Union{Satisfy,Scan,Token,Tokens,Fail}, ::Vector{Bool}) = false
-can_match_epsilon(x::Epsilon, ::Vector{Bool}) = true
+can_match_epsilon(x::Union{Epsilon,EndOfInput}, ::Vector{Bool}) = true
 can_match_epsilon(x::Seq, ch::Vector{Bool}) = all(ch)
 can_match_epsilon(x::First, ch::Vector{Bool}) =
     isempty(ch) ? false :
@@ -236,16 +237,21 @@ end
 # Epsilon matches
 #
 
-match_epsilon!(x::Clause, id::Int, pos::Int, st::ParserState) =
+match_epsilon!(x::Clause, id::Int, pos::Int, st::ParserState)::MatchResult =
     new_match!(Match(id, pos, prevind(st.input, pos), 0, submatch_empty(st)), st)
 
-function match_epsilon!(x::FollowedBy, id::Int, pos::Int, st::ParserState)
+function match_epsilon!(x::EndOfInput, id::Int, pos::Int, st::ParserState)::MatchResult
+    pos <= lastindex(st.input) ? 0 :
+    new_match!(Match(id, pos, prevind(st.input, pos), 0, submatch_empty(st)), st)
+end
+
+function match_epsilon!(x::FollowedBy, id::Int, pos::Int, st::ParserState)::MatchResult
     mid = lookup_best_match_id!(pos, x.follow, st)
     mid == 0 ? 0 :
     new_match!(Match(id, pos, prevind(st.input, pos), 1, submatch_record!(st, mid)), st)
 end
 
-function match_epsilon!(x::NotFollowedBy, id::Int, pos::Int, st::ParserState)
+function match_epsilon!(x::NotFollowedBy, id::Int, pos::Int, st::ParserState)::MatchResult
     # This might technically cause infinite recursion, byt a cycle of
     # NotFollowedBy clauses is disallowed by the error thrown by
     # can_match_epsilon(::NotFollowedBy, ...)

src/frontend.jl

@@ -48,6 +48,17 @@ function tokens(xs::I) where {I}
     Tokens{Any,eltype(I),I}(xs)
 end
 
+"""
+    end_of_input :: Clause
+
+An [`EndOfInput`](@ref) clause. Translate to strongly typed grammar with [`flatten`](@ref).
+
+# Example
+
+    whole_file = first(:file_contents, end_of_input)
+"""
+const end_of_input = EndOfInput{Any,Any}()
+
 """
     epsilon :: Clause
 

src/parse.jl

@@ -15,6 +15,8 @@ function lookup_best_match_id!(
             return match_epsilon!(cls, clause, pos, st)
         elseif cls isa NotFollowedBy{Int,T}
             return match_epsilon!(cls, clause, pos, st)
+        elseif cls isa EndOfInput{Int,T}
+            return match_epsilon!(cls, clause, pos, st)
         elseif cls isa Epsilon{Int,T}
             return match_epsilon!(cls, clause, pos, st)
         elseif cls isa Many{Int,T}

src/structs.jl

@@ -16,6 +16,7 @@ Currently implemented clauses:
 - [`Tokens`](@ref)
 - [`Epsilon`](@ref)
 - [`Fail`](@ref)
+- [`EndOfInput`](@ref)
 - [`Seq`](@ref)
 - [`First`](@ref)
 - [`NotFollowedBy`](@ref)
@@ -87,6 +88,13 @@ end
 """
 $(TYPEDEF)
 
+Matches at the end of the input.
+"""
+struct EndOfInput{G,T} <: Clause{G,T} end
+
+"""
+$(TYPEDEF)
+
 An always-succeeding epsilon match.
 """
 struct Epsilon{G,T} <: Clause{G,T} end

test/clauses.jl

@@ -91,9 +91,11 @@ end
 end
 
 @testset "Corner-case epsilon matches" begin
+    str = "whateveρ"
+
     rules = Dict(:x => P.followed_by(P.epsilon))
 
-    p = P.parse(P.make_grammar([:x], P.flatten(rules, Char)), "whatever")
+    p = P.parse(P.make_grammar([:x], P.flatten(rules, Char)), str)
 
     @test P.find_match_at!(p, :x, 1) != 0
     @test P.find_match_at!(p, :x, 8) != 0
@@ -102,7 +104,14 @@ end
 
     # tie epsilon match
     rules = Dict(:x => P.tie(P.epsilon))
-    p = P.parse(P.make_grammar([:x], P.flatten(rules, Char)), "whatever")
+    p = P.parse(P.make_grammar([:x], P.flatten(rules, Char)), str)
 
     @test P.traverse_match(p, P.find_match_at!(p, :x, 1)) == :(x())
+
+    rules = Dict(:x => P.end_of_input)
+    p = P.parse(P.make_grammar([:x], P.flatten(rules, Char)), str)
+
+    @test P.find_match_at!(p, :x, firstindex(str)) == 0
+    @test P.find_match_at!(p, :x, lastindex(str)) == 0
+    @test P.find_match_at!(p, :x, nextind(str, lastindex(str))) != 0
 end