cache/filecache: Split implementation and config into separate files

This commit is contained in:
Bjørn Erik Pedersen 2018-11-14 10:44:04 +01:00
parent e4b2572880
commit 17d7ecde2b
No known key found for this signature in database
GPG key ID: 330E6E2BD4859D8F
4 changed files with 299 additions and 251 deletions

View file

@ -17,7 +17,6 @@ import (
"bytes"
"io"
"io/ioutil"
"path"
"path/filepath"
"strings"
"time"
@ -28,56 +27,10 @@ import (
"github.com/gohugoio/hugo/hugolib/paths"
"github.com/pkg/errors"
"github.com/BurntSushi/locker"
"github.com/bep/mapstructure"
"github.com/spf13/afero"
)
const (
cachesConfigKey = "caches"
resourcesGenDir = ":resourceDir/_gen"
)
var defaultCacheConfig = cacheConfig{
MaxAge: -1, // Never expire
Dir: ":cacheDir",
}
const (
cacheKeyGetJSON = "getjson"
cacheKeyGetCSV = "getcsv"
cacheKeyImages = "images"
cacheKeyAssets = "assets"
)
var defaultCacheConfigs = map[string]cacheConfig{
cacheKeyGetJSON: defaultCacheConfig,
cacheKeyGetCSV: defaultCacheConfig,
cacheKeyImages: cacheConfig{
MaxAge: -1,
Dir: resourcesGenDir,
},
cacheKeyAssets: cacheConfig{
MaxAge: -1,
Dir: resourcesGenDir,
},
}
type cachesConfig map[string]cacheConfig
type cacheConfig struct {
// Max age of cache entries in this cache. Any items older than this will
// be removed and not returned from the cache.
// -1 means forever, 0 means cache is disabled.
MaxAge int
// The directory where files are stored.
Dir string
}
// Cache caches a set of files in a directory. This is usually a file on
// disk, but since this is backed by an Afero file system, it can be anything.
type Cache struct {
@ -316,26 +269,6 @@ func (f Caches) Get(name string) *Cache {
return f[strings.ToLower(name)]
}
// GetJSONCache gets the file cache for getJSON.
func (f Caches) GetJSONCache() *Cache {
return f[cacheKeyGetJSON]
}
// GetCSVCache gets the file cache for getCSV.
func (f Caches) GetCSVCache() *Cache {
return f[cacheKeyGetCSV]
}
// ImageCache gets the file cache for processed images.
func (f Caches) ImageCache() *Cache {
return f[cacheKeyImages]
}
// AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
func (f Caches) AssetsCache() *Cache {
return f[cacheKeyAssets]
}
// NewCachesFromPaths creates a new set of file caches from the given
// configuration.
func NewCachesFromPaths(p *paths.Paths) (Caches, error) {
@ -359,84 +292,6 @@ func NewCachesFromPaths(p *paths.Paths) (Caches, error) {
return m, nil
}
func decodeConfig(p *paths.Paths) (cachesConfig, error) {
c := make(cachesConfig)
valid := make(map[string]bool)
// Add defaults
for k, v := range defaultCacheConfigs {
c[k] = v
valid[k] = true
}
cfg := p.Cfg
m := cfg.GetStringMap(cachesConfigKey)
_, isOsFs := p.Fs.Source.(*afero.OsFs)
for k, v := range m {
cc := defaultCacheConfig
if err := mapstructure.WeakDecode(v, &cc); err != nil {
return nil, err
}
if cc.Dir == "" {
return c, errors.New("must provide cache Dir")
}
name := strings.ToLower(k)
if !valid[name] {
return nil, errors.Errorf("%q is not a valid cache name", name)
}
c[name] = cc
}
// This is a very old flag in Hugo, but we need to respect it.
disabled := cfg.GetBool("ignoreCache")
for k, v := range c {
v.Dir = filepath.Clean(v.Dir)
dir := filepath.ToSlash(v.Dir)
parts := strings.Split(dir, "/")
first := parts[0]
if strings.HasPrefix(first, ":") {
resolved, err := resolveDirPlaceholder(p, first)
if err != nil {
return c, err
}
resolved = filepath.ToSlash(resolved)
v.Dir = filepath.FromSlash(path.Join((append([]string{resolved}, parts[1:]...))...))
} else if isOsFs && !path.IsAbs(dir) {
return c, errors.Errorf("%q must either start with a placeholder (e.g. :cacheDir, :resourceDir) or be absolute", v.Dir)
}
if disabled {
v.MaxAge = 0
}
c[k] = v
}
return c, nil
}
// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
func resolveDirPlaceholder(p *paths.Paths, placeholder string) (string, error) {
switch strings.ToLower(placeholder) {
case ":resourcedir":
return p.AbsResourcesDir, nil
case ":cachedir":
return helpers.GetCacheDir(p.Fs.Source, p.Cfg)
}
return "", errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
}
func cleanID(name string) string {
return filepath.Clean(name)
}

168
cache/filecache/filecache_config.go vendored Normal file
View file

@ -0,0 +1,168 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package filecache
import (
"path"
"path/filepath"
"strings"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugolib/paths"
"github.com/bep/mapstructure"
"github.com/pkg/errors"
"github.com/spf13/afero"
)
const (
cachesConfigKey = "caches"
resourcesGenDir = ":resourceDir/_gen"
)
var defaultCacheConfig = cacheConfig{
MaxAge: -1, // Never expire
Dir: ":cacheDir",
}
const (
cacheKeyGetJSON = "getjson"
cacheKeyGetCSV = "getcsv"
cacheKeyImages = "images"
cacheKeyAssets = "assets"
)
var defaultCacheConfigs = map[string]cacheConfig{
cacheKeyGetJSON: defaultCacheConfig,
cacheKeyGetCSV: defaultCacheConfig,
cacheKeyImages: cacheConfig{
MaxAge: -1,
Dir: resourcesGenDir,
},
cacheKeyAssets: cacheConfig{
MaxAge: -1,
Dir: resourcesGenDir,
},
}
type cachesConfig map[string]cacheConfig
type cacheConfig struct {
// Max age of cache entries in this cache. Any items older than this will
// be removed and not returned from the cache.
// -1 means forever, 0 means cache is disabled.
MaxAge int
// The directory where files are stored.
Dir string
}
// GetJSONCache gets the file cache for getJSON.
func (f Caches) GetJSONCache() *Cache {
return f[cacheKeyGetJSON]
}
// GetCSVCache gets the file cache for getCSV.
func (f Caches) GetCSVCache() *Cache {
return f[cacheKeyGetCSV]
}
// ImageCache gets the file cache for processed images.
func (f Caches) ImageCache() *Cache {
return f[cacheKeyImages]
}
// AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
func (f Caches) AssetsCache() *Cache {
return f[cacheKeyAssets]
}
func decodeConfig(p *paths.Paths) (cachesConfig, error) {
c := make(cachesConfig)
valid := make(map[string]bool)
// Add defaults
for k, v := range defaultCacheConfigs {
c[k] = v
valid[k] = true
}
cfg := p.Cfg
m := cfg.GetStringMap(cachesConfigKey)
_, isOsFs := p.Fs.Source.(*afero.OsFs)
for k, v := range m {
cc := defaultCacheConfig
if err := mapstructure.WeakDecode(v, &cc); err != nil {
return nil, err
}
if cc.Dir == "" {
return c, errors.New("must provide cache Dir")
}
name := strings.ToLower(k)
if !valid[name] {
return nil, errors.Errorf("%q is not a valid cache name", name)
}
c[name] = cc
}
// This is a very old flag in Hugo, but we need to respect it.
disabled := cfg.GetBool("ignoreCache")
for k, v := range c {
v.Dir = filepath.Clean(v.Dir)
dir := filepath.ToSlash(v.Dir)
parts := strings.Split(dir, "/")
first := parts[0]
if strings.HasPrefix(first, ":") {
resolved, err := resolveDirPlaceholder(p, first)
if err != nil {
return c, err
}
resolved = filepath.ToSlash(resolved)
v.Dir = filepath.FromSlash(path.Join((append([]string{resolved}, parts[1:]...))...))
} else if isOsFs && !path.IsAbs(dir) {
return c, errors.Errorf("%q must either start with a placeholder (e.g. :cacheDir, :resourceDir) or be absolute", v.Dir)
}
if disabled {
v.MaxAge = 0
}
c[k] = v
}
return c, nil
}
// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
func resolveDirPlaceholder(p *paths.Paths, placeholder string) (string, error) {
switch strings.ToLower(placeholder) {
case ":resourcedir":
return p.AbsResourcesDir, nil
case ":cachedir":
return helpers.GetCacheDir(p.Fs.Source, p.Cfg)
}
return "", errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
}

131
cache/filecache/filecache_config_test.go vendored Normal file
View file

@ -0,0 +1,131 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package filecache
import (
"path/filepath"
"runtime"
"testing"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugolib/paths"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
func TestDecodeConfig(t *testing.T) {
t.Parallel()
assert := require.New(t)
configStr := `
[caches]
[caches.getJSON]
maxAge = 1234
dir = "/path/to/c1"
[caches.getCSV]
maxAge = 3456
dir = "/path/to/c2"
[caches.images]
dir = "/path/to/c3"
`
cfg, err := config.FromConfigString(configStr, "toml")
assert.NoError(err)
fs := hugofs.NewMem(cfg)
p, err := paths.New(fs, cfg)
assert.NoError(err)
decoded, err := decodeConfig(p)
assert.NoError(err)
assert.Equal(4, len(decoded))
c2 := decoded["getcsv"]
assert.Equal(3456, c2.MaxAge)
assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir)
c3 := decoded["images"]
assert.Equal(-1, c3.MaxAge)
assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir)
}
func TestDecodeConfigIgnoreCache(t *testing.T) {
t.Parallel()
assert := require.New(t)
configStr := `
ignoreCache = true
[caches]
[caches.getJSON]
maxAge = 1234
dir = "/path/to/c1"
[caches.getCSV]
maxAge = 3456
dir = "/path/to/c2"
[caches.images]
dir = "/path/to/c3"
`
cfg, err := config.FromConfigString(configStr, "toml")
assert.NoError(err)
fs := hugofs.NewMem(cfg)
p, err := paths.New(fs, cfg)
assert.NoError(err)
decoded, err := decodeConfig(p)
assert.NoError(err)
assert.Equal(4, len(decoded))
for _, v := range decoded {
assert.Equal(0, v.MaxAge)
}
}
func TestDecodeConfigDefault(t *testing.T) {
assert := require.New(t)
cfg := viper.New()
if runtime.GOOS == "windows" {
cfg.Set("resourceDir", "c:\\cache\\resources")
cfg.Set("cacheDir", "c:\\cache\\thecache")
} else {
cfg.Set("resourceDir", "/cache/resources")
cfg.Set("cacheDir", "/cache/thecache")
}
fs := hugofs.NewMem(cfg)
p, err := paths.New(fs, cfg)
assert.NoError(err)
decoded, err := decodeConfig(p)
assert.NoError(err)
assert.Equal(4, len(decoded))
if runtime.GOOS == "windows" {
assert.Equal("c:\\cache\\resources\\_gen", decoded[cacheKeyImages].Dir)
} else {
assert.Equal("/cache/resources/_gen", decoded[cacheKeyImages].Dir)
}
}

View file

@ -19,7 +19,6 @@ import (
"io/ioutil"
"path/filepath"
"regexp"
"runtime"
"strings"
"sync"
"testing"
@ -31,7 +30,6 @@ import (
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugolib/paths"
"github.com/spf13/afero"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
@ -200,107 +198,3 @@ dir = "/cache/c"
}
wg.Wait()
}
func TestDecodeConfig(t *testing.T) {
t.Parallel()
assert := require.New(t)
configStr := `
[caches]
[caches.getJSON]
maxAge = 1234
dir = "/path/to/c1"
[caches.getCSV]
maxAge = 3456
dir = "/path/to/c2"
[caches.images]
dir = "/path/to/c3"
`
cfg, err := config.FromConfigString(configStr, "toml")
assert.NoError(err)
fs := hugofs.NewMem(cfg)
p, err := paths.New(fs, cfg)
assert.NoError(err)
decoded, err := decodeConfig(p)
assert.NoError(err)
assert.Equal(4, len(decoded))
c2 := decoded["getcsv"]
assert.Equal(3456, c2.MaxAge)
assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir)
c3 := decoded["images"]
assert.Equal(-1, c3.MaxAge)
assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir)
}
func TestDecodeConfigIgnoreCache(t *testing.T) {
t.Parallel()
assert := require.New(t)
configStr := `
ignoreCache = true
[caches]
[caches.getJSON]
maxAge = 1234
dir = "/path/to/c1"
[caches.getCSV]
maxAge = 3456
dir = "/path/to/c2"
[caches.images]
dir = "/path/to/c3"
`
cfg, err := config.FromConfigString(configStr, "toml")
assert.NoError(err)
fs := hugofs.NewMem(cfg)
p, err := paths.New(fs, cfg)
assert.NoError(err)
decoded, err := decodeConfig(p)
assert.NoError(err)
assert.Equal(4, len(decoded))
for _, v := range decoded {
assert.Equal(0, v.MaxAge)
}
}
func TestDecodeConfigDefault(t *testing.T) {
assert := require.New(t)
cfg := viper.New()
if runtime.GOOS == "windows" {
cfg.Set("resourceDir", "c:\\cache\\resources")
cfg.Set("cacheDir", "c:\\cache\\thecache")
} else {
cfg.Set("resourceDir", "/cache/resources")
cfg.Set("cacheDir", "/cache/thecache")
}
fs := hugofs.NewMem(cfg)
p, err := paths.New(fs, cfg)
assert.NoError(err)
decoded, err := decodeConfig(p)
assert.NoError(err)
assert.Equal(4, len(decoded))
if runtime.GOOS == "windows" {
assert.Equal("c:\\cache\\resources\\_gen", decoded[cacheKeyImages].Dir)
} else {
assert.Equal("/cache/resources/_gen", decoded[cacheKeyImages].Dir)
}
}