cache/filecache: Recover from file corruption

Fixes #6401
This commit is contained in:
Bjørn Erik Pedersen 2019-10-21 09:37:46 +02:00
parent 4b286b9d27
commit 180195aa34
No known key found for this signature in database
GPG key ID: 330E6E2BD4859D8F
2 changed files with 61 additions and 1 deletions

View file

@ -15,6 +15,7 @@ package filecache
import ( import (
"bytes" "bytes"
"errors"
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
@ -31,6 +32,9 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
) )
// ErrFatal can be used to signal an unrecoverable error.
var ErrFatal = errors.New("fatal filecache error")
const ( const (
filecacheRootDirname = "filecache" filecacheRootDirname = "filecache"
) )
@ -137,8 +141,14 @@ func (c *Cache) ReadOrCreate(id string,
if r := c.getOrRemove(id); r != nil { if r := c.getOrRemove(id); r != nil {
err = read(info, r) err = read(info, r)
defer r.Close() defer r.Close()
if err == nil || err == ErrFatal {
// See https://github.com/gohugoio/hugo/issues/6401
// To recover from file corruption we handle read errors
// as the cache item was not found.
// Any file permission issue will also fail in the next step.
return return
} }
}
f, err := helpers.OpenFileForWriting(c.Fs, id) f, err := helpers.OpenFileForWriting(c.Fs, id)
if err != nil { if err != nil {

View file

@ -14,6 +14,7 @@
package filecache package filecache
import ( import (
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
@ -243,6 +244,55 @@ dir = "/cache/c"
wg.Wait() wg.Wait()
} }
func TestFileCacheReadOrCreateErrorInRead(t *testing.T) {
t.Parallel()
c := qt.New(t)
var result string
rf := func(failLevel int) func(info ItemInfo, r io.Reader) error {
return func(info ItemInfo, r io.Reader) error {
if failLevel > 0 {
if failLevel > 1 {
return ErrFatal
}
return errors.New("fail")
}
b, _ := ioutil.ReadAll(r)
result = string(b)
return nil
}
}
bf := func(s string) func(info ItemInfo, w io.WriteCloser) error {
return func(info ItemInfo, w io.WriteCloser) error {
defer w.Close()
result = s
_, err := w.Write([]byte(s))
return err
}
}
cache := NewCache(afero.NewMemMapFs(), 100*time.Hour, "")
const id = "a32"
_, err := cache.ReadOrCreate(id, rf(0), bf("v1"))
c.Assert(err, qt.IsNil)
c.Assert(result, qt.Equals, "v1")
_, err = cache.ReadOrCreate(id, rf(0), bf("v2"))
c.Assert(err, qt.IsNil)
c.Assert(result, qt.Equals, "v1")
_, err = cache.ReadOrCreate(id, rf(1), bf("v3"))
c.Assert(err, qt.IsNil)
c.Assert(result, qt.Equals, "v3")
_, err = cache.ReadOrCreate(id, rf(2), bf("v3"))
c.Assert(err, qt.Equals, ErrFatal)
}
func TestCleanID(t *testing.T) { func TestCleanID(t *testing.T) {
c := qt.New(t) c := qt.New(t)
c.Assert(cleanID(filepath.FromSlash("/a/b//c.txt")), qt.Equals, filepath.FromSlash("a/b/c.txt")) c.Assert(cleanID(filepath.FromSlash("/a/b//c.txt")), qt.Equals, filepath.FromSlash("a/b/c.txt"))