From 78e9229c52dfdaf335673b171a3d66fc1ace8d9e Mon Sep 17 00:00:00 2001 From: Tatsushi Demachi Date: Sat, 10 Jan 2015 16:15:51 +0900 Subject: [PATCH] Fix "hugo new" EOF error with an archetype file without the final EOL This rewrites `extractFrontMatterDelims` function to make it work with an archetype file without the final EOL and adds more detailed error messages and comments. It also removes `matches` and `matches_quick` functions which aren't called anywhere. --- hugolib/page_test.go | 2 +- parser/page.go | 146 ++++++++++++++----------------- parser/parse_frontmatter_test.go | 3 +- 3 files changed, 70 insertions(+), 81 deletions(-) diff --git a/hugolib/page_test.go b/hugolib/page_test.go index f6ff83aaa..887bd1016 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -447,7 +447,7 @@ func TestDegenerateInvalidFrontMatterShortDelim(t *testing.T) { r string err string }{ - {INVALID_FRONT_MATTER_SHORT_DELIM_ENDING, "Unable to read frontmatter at filepos 45: EOF"}, + {INVALID_FRONT_MATTER_SHORT_DELIM_ENDING, "unable to read frontmatter at filepos 45: EOF"}, } for _, test := range tests { diff --git a/parser/page.go b/parser/page.go index a3c0ec7d7..e384d3858 100644 --- a/parser/page.go +++ b/parser/page.go @@ -159,17 +159,13 @@ func isFrontMatterDelim(data []byte) bool { func determineDelims(firstLine []byte) (left, right []byte) { switch len(firstLine) { + case 5: + fallthrough case 4: if firstLine[0] == YAML_LEAD[0] { - return []byte(YAML_DELIM_UNIX), []byte(YAML_DELIM_UNIX) + return []byte(YAML_DELIM), []byte(YAML_DELIM) } - return []byte(TOML_DELIM_UNIX), []byte(TOML_DELIM_UNIX) - - case 5: - if firstLine[0] == YAML_LEAD[0] { - return []byte(YAML_DELIM_DOS), []byte(YAML_DELIM_DOS) - } - return []byte(TOML_DELIM_DOS), []byte(TOML_DELIM_DOS) + return []byte(TOML_DELIM), []byte(TOML_DELIM) case 3: fallthrough case 2: @@ -181,104 +177,98 @@ func determineDelims(firstLine []byte) (left, right []byte) { } } +// extractFrontMatterDelims takes a frontmatter from the content bufio.Reader. +// Begining white spaces of the bufio.Reader must be trimmed before call this +// function. func extractFrontMatterDelims(r *bufio.Reader, left, right []byte) (fm FrontMatter, err error) { var ( c byte - level int = 0 - bytesRead int = 0 - sameDelim = bytes.Equal(left, right) + buf bytes.Buffer + level int = 0 + sameDelim bool = bytes.Equal(left, right) ) - wr := new(bytes.Buffer) + // Frontmatter must start with a delimiter. To check it first, + // pre-reads beginning delimiter length - 1 bytes from Reader + for i := 0; i < len(left)-1; i++ { + if c, err = r.ReadByte(); err != nil { + return nil, fmt.Errorf("unable to read frontmatter at filepos %d: %s", buf.Len(), err) + } + if err = buf.WriteByte(c); err != nil { + return nil, err + } + } + + // Reads a character from Reader one by one and checks it matches the + // last character of one of delemiters to find the last character of + // frontmatter. If it matches, makes sure it contains the delimiter + // and if so, also checks it is followed by CR+LF or LF when YAML, + // TOML case. In JSON case, nested delimiters must be parsed and it + // is expected that the delimiter only contains one character. for { if c, err = r.ReadByte(); err != nil { - return nil, fmt.Errorf("Unable to read frontmatter at filepos %d: %s", bytesRead, err) + return nil, fmt.Errorf("unable to read frontmatter at filepos %d: %s", buf.Len(), err) + } + if err = buf.WriteByte(c); err != nil { + return nil, err } - bytesRead += 1 switch c { - case left[0]: - var ( - buf []byte = []byte{c} - remaining []byte - ) - - if remaining, err = r.Peek(len(left) - 1); err != nil { - return nil, err - } - - buf = append(buf, remaining...) - - if bytes.Equal(buf, left) { - if sameDelim { + case left[len(left)-1]: + if sameDelim { // YAML, TOML case + if bytes.HasSuffix(buf.Bytes(), left) { + c, err = r.ReadByte() + if err != nil { + // It is ok that the end delimiter ends with EOF + if err != io.EOF || level != 1 { + return nil, fmt.Errorf("unable to read frontmatter at filepos %d: %s", buf.Len(), err) + } + } else { + switch c { + case '\n': + // ok + case '\r': + if err = buf.WriteByte(c); err != nil { + return nil, err + } + if c, err = r.ReadByte(); err != nil { + return nil, fmt.Errorf("unable to read frontmatter at filepos %d: %s", buf.Len(), err) + } + if c != '\n' { + return nil, fmt.Errorf("frontmatter delimiter must be followed by CR+LF or LF but those can't be found at filepos %d", buf.Len()) + } + default: + return nil, fmt.Errorf("frontmatter delimiter must be followed by CR+LF or LF but those can't be found at filepos %d", buf.Len()) + } + if err = buf.WriteByte(c); err != nil { + return nil, err + } + } if level == 0 { level = 1 } else { level = 0 } - } else { - level += 1 } + } else { // JSON case + level++ } - - if _, err = wr.Write([]byte{c}); err != nil { - return nil, err - } - - if level == 0 { - if _, err = r.Read(remaining); err != nil { - return nil, err - } - if _, err = wr.Write(remaining); err != nil { - return nil, err - } - } - case right[0]: - match, err := matches(r, wr, []byte{c}, right) - if err != nil { - return nil, err - } - if match { - level -= 1 - } - default: - if err = wr.WriteByte(c); err != nil { - return nil, err - } + case right[len(right)-1]: // JSON case only reaches here + level-- } - if level == 0 && !unicode.IsSpace(rune(c)) { + if level == 0 { + // Consumes white spaces immediately behind frontmatter if err = chompWhitespace(r); err != nil { if err != io.EOF { return nil, err } } - return wr.Bytes(), nil + return buf.Bytes(), nil } } } -func matches_quick(buf, expected []byte) (ok bool, err error) { - return bytes.Equal(expected, buf), nil -} - -func matches(r *bufio.Reader, wr io.Writer, c, expected []byte) (ok bool, err error) { - if len(expected) == 1 { - if _, err = wr.Write(c); err != nil { - return - } - return bytes.Equal(c, expected), nil - } - - buf := make([]byte, len(expected)-1) - if buf, err = r.Peek(len(expected) - 1); err != nil { - return - } - - buf = append(c, buf...) - return bytes.Equal(expected, buf), nil -} - func extractContent(r io.Reader) (content Content, err error) { wr := new(bytes.Buffer) if _, err = wr.ReadFrom(r); err != nil { diff --git a/parser/parse_frontmatter_test.go b/parser/parse_frontmatter_test.go index ab7d78eb0..deb885f12 100644 --- a/parser/parse_frontmatter_test.go +++ b/parser/parse_frontmatter_test.go @@ -54,7 +54,6 @@ func TestDegenerateCreatePageFrom(t *testing.T) { }{ {CONTENT_MISSING_END_FM_DELIM}, {CONTENT_INCOMPLETE_END_FM_DELIM}, - {CONTENT_FM_NO_DOC}, } for _, test := range tests { @@ -230,6 +229,7 @@ func TestExtractFrontMatter(t *testing.T) { {"---\nfoobar\nbarfoo\nfizbaz\n", nil, false}, {"---\nblar\n-\n", nil, false}, {"---\nralb\n---\n", []byte("---\nralb\n---\n"), true}, + {"---\neof\n---", []byte("---\neof\n---"), true}, {"---\nminc\n---\ncontent", []byte("---\nminc\n---\n"), true}, {"---\ncnim\n---\ncontent\n", []byte("---\ncnim\n---\n"), true}, {"---\ntitle: slug doc 2\nslug: slug-doc-2\n---\ncontent\n", []byte("---\ntitle: slug doc 2\nslug: slug-doc-2\n---\n"), true}, @@ -274,7 +274,6 @@ func TestExtractFrontMatterDelim(t *testing.T) { {"", "", errExpected}, {"{", "", errExpected}, {"{}", "{}", noErrExpected}, - {" {}", " {}", noErrExpected}, {"{} ", "{}", noErrExpected}, {"{ } ", "{ }", noErrExpected}, {"{ { }", "", errExpected},