Always use content to resolve content type in resources.GetRemote

This is a security hardening measure; don't trust the URL extension or any `Content-Type`/`Content-Disposition` header on its own, always look at the file content using Go's `http.DetectContentType`.

This commit also adds ttf and otf media type definitions to Hugo.

Fixes #9302
Fixes #9301
This commit is contained in:
Bjørn Erik Pedersen 2021-12-16 15:12:13 +01:00
parent 22ef5da20d
commit 44954497bc
No known key found for this signature in database
GPG key ID: 330E6E2BD4859D8F
26 changed files with 378 additions and 49 deletions

View file

@ -17,6 +17,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"net/http"
"sort" "sort"
"strings" "strings"
@ -60,6 +61,42 @@ type SuffixInfo struct {
FullSuffix string `json:"fullSuffix"` FullSuffix string `json:"fullSuffix"`
} }
// FromContent resolve the Type primarily using http.DetectContentType.
// If http.DetectContentType resolves to application/octet-stream, a zero Type is returned.
// If http.DetectContentType resolves to text/plain or application/xml, we try to get more specific using types and ext.
func FromContent(types Types, ext string, content []byte) Type {
ext = strings.TrimPrefix(ext, ".")
t := strings.Split(http.DetectContentType(content), ";")[0]
var m Type
if t == "application/octet-stream" {
return m
}
var found bool
m, found = types.GetByType(t)
if !found {
if t == "text/xml" {
// This is how it's configured in Hugo by default.
m, found = types.GetByType("application/xml")
}
}
if !found || ext == "" {
return m
}
if m.Type() == "text/plain" || m.Type() == "application/xml" {
// http.DetectContentType isn't brilliant when it comes to common text formats, so we need to do better.
// For now we say that if it's detected to be a text format and the extension/content type in header reports
// it to be a text format, then we use that.
mm, _, found := types.GetFirstBySuffix(ext)
if found && mm.IsText() {
return mm
}
}
return m
}
// FromStringAndExt creates a Type from a MIME string and a given extension. // FromStringAndExt creates a Type from a MIME string and a given extension.
func FromStringAndExt(t, ext string) (Type, error) { func FromStringAndExt(t, ext string) (Type, error) {
tp, err := fromString(t) tp, err := fromString(t)
@ -122,6 +159,21 @@ func (m Type) Suffixes() []string {
return strings.Split(m.suffixesCSV, ",") return strings.Split(m.suffixesCSV, ",")
} }
// IsText returns whether this Type is a text format.
// Note that this may currently return false negatives.
// TODO(bep) improve
func (m Type) IsText() bool {
if m.MainType == "text" {
return true
}
switch m.SubType {
case "javascript", "json", "rss", "xml", "svg", TOMLType.SubType, YAMLType.SubType:
return true
}
return false
}
func (m *Type) init() { func (m *Type) init() {
m.FirstSuffix.FullSuffix = "" m.FirstSuffix.FullSuffix = ""
m.FirstSuffix.Suffix = "" m.FirstSuffix.Suffix = ""
@ -183,6 +235,10 @@ var (
BMPType = newMediaType("image", "bmp", []string{"bmp"}) BMPType = newMediaType("image", "bmp", []string{"bmp"})
WEBPType = newMediaType("image", "webp", []string{"webp"}) WEBPType = newMediaType("image", "webp", []string{"webp"})
// Common font types
TrueTypeFontType = newMediaType("font", "ttf", []string{"ttf"})
OpenTypeFontType = newMediaType("font", "otf", []string{"otf"})
// Common video types // Common video types
AVIType = newMediaType("video", "x-msvideo", []string{"avi"}) AVIType = newMediaType("video", "x-msvideo", []string{"avi"})
MPEGType = newMediaType("video", "mpeg", []string{"mpg", "mpeg"}) MPEGType = newMediaType("video", "mpeg", []string{"mpg", "mpeg"})
@ -224,6 +280,8 @@ var DefaultTypes = Types{
OGGType, OGGType,
WEBMType, WEBMType,
GPPType, GPPType,
OpenTypeFontType,
TrueTypeFontType,
} }
func init() { func init() {

View file

@ -15,10 +15,14 @@ package media
import ( import (
"encoding/json" "encoding/json"
"io/ioutil"
"path/filepath"
"sort" "sort"
"strings"
"testing" "testing"
qt "github.com/frankban/quicktest" qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/common/paths"
) )
func TestDefaultTypes(t *testing.T) { func TestDefaultTypes(t *testing.T) {
@ -47,6 +51,8 @@ func TestDefaultTypes(t *testing.T) {
{XMLType, "application", "xml", "xml", "application/xml", "application/xml"}, {XMLType, "application", "xml", "xml", "application/xml", "application/xml"},
{TOMLType, "application", "toml", "toml", "application/toml", "application/toml"}, {TOMLType, "application", "toml", "toml", "application/toml", "application/toml"},
{YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"}, {YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"},
{TrueTypeFontType, "font", "ttf", "ttf", "font/ttf", "font/ttf"},
{OpenTypeFontType, "font", "otf", "otf", "font/otf", "font/otf"},
} { } {
c.Assert(test.tp.MainType, qt.Equals, test.expectedMainType) c.Assert(test.tp.MainType, qt.Equals, test.expectedMainType)
c.Assert(test.tp.SubType, qt.Equals, test.expectedSubType) c.Assert(test.tp.SubType, qt.Equals, test.expectedSubType)
@ -56,7 +62,7 @@ func TestDefaultTypes(t *testing.T) {
} }
c.Assert(len(DefaultTypes), qt.Equals, 28) c.Assert(len(DefaultTypes), qt.Equals, 30)
} }
func TestGetByType(t *testing.T) { func TestGetByType(t *testing.T) {
@ -175,6 +181,26 @@ func TestFromExtensionMultipleSuffixes(t *testing.T) {
} }
func TestFromContent(t *testing.T) {
c := qt.New(t)
files, err := filepath.Glob("./testdata/resource.*")
c.Assert(err, qt.IsNil)
mtypes := DefaultTypes
for _, filename := range files {
c.Run(filepath.Base(filename), func(c *qt.C) {
content, err := ioutil.ReadFile(filename)
c.Assert(err, qt.IsNil)
ext := strings.TrimPrefix(paths.Ext(filename), ".")
expected, _, found := mtypes.GetFirstBySuffix(ext)
c.Assert(found, qt.IsTrue)
got := FromContent(mtypes, ext, content)
c.Assert(got, qt.Equals, expected)
})
}
}
func TestDecodeTypes(t *testing.T) { func TestDecodeTypes(t *testing.T) {
c := qt.New(t) c := qt.New(t)

BIN
media/testdata/reosurce.otf vendored Normal file

Binary file not shown.

8
media/testdata/resource.css vendored Normal file
View file

@ -0,0 +1,8 @@
body {
background-color: lightblue;
}
h1 {
color: navy;
margin-left: 20px;
}

130
media/testdata/resource.csv vendored Normal file
View file

@ -0,0 +1,130 @@
"LatD", "LatM", "LatS", "NS", "LonD", "LonM", "LonS", "EW", "City", "State"
41, 5, 59, "N", 80, 39, 0, "W", "Youngstown", OH
42, 52, 48, "N", 97, 23, 23, "W", "Yankton", SD
46, 35, 59, "N", 120, 30, 36, "W", "Yakima", WA
42, 16, 12, "N", 71, 48, 0, "W", "Worcester", MA
43, 37, 48, "N", 89, 46, 11, "W", "Wisconsin Dells", WI
36, 5, 59, "N", 80, 15, 0, "W", "Winston-Salem", NC
49, 52, 48, "N", 97, 9, 0, "W", "Winnipeg", MB
39, 11, 23, "N", 78, 9, 36, "W", "Winchester", VA
34, 14, 24, "N", 77, 55, 11, "W", "Wilmington", NC
39, 45, 0, "N", 75, 33, 0, "W", "Wilmington", DE
48, 9, 0, "N", 103, 37, 12, "W", "Williston", ND
41, 15, 0, "N", 77, 0, 0, "W", "Williamsport", PA
37, 40, 48, "N", 82, 16, 47, "W", "Williamson", WV
33, 54, 0, "N", 98, 29, 23, "W", "Wichita Falls", TX
37, 41, 23, "N", 97, 20, 23, "W", "Wichita", KS
40, 4, 11, "N", 80, 43, 12, "W", "Wheeling", WV
26, 43, 11, "N", 80, 3, 0, "W", "West Palm Beach", FL
47, 25, 11, "N", 120, 19, 11, "W", "Wenatchee", WA
41, 25, 11, "N", 122, 23, 23, "W", "Weed", CA
31, 13, 11, "N", 82, 20, 59, "W", "Waycross", GA
44, 57, 35, "N", 89, 38, 23, "W", "Wausau", WI
42, 21, 36, "N", 87, 49, 48, "W", "Waukegan", IL
44, 54, 0, "N", 97, 6, 36, "W", "Watertown", SD
43, 58, 47, "N", 75, 55, 11, "W", "Watertown", NY
42, 30, 0, "N", 92, 20, 23, "W", "Waterloo", IA
41, 32, 59, "N", 73, 3, 0, "W", "Waterbury", CT
38, 53, 23, "N", 77, 1, 47, "W", "Washington", DC
41, 50, 59, "N", 79, 8, 23, "W", "Warren", PA
46, 4, 11, "N", 118, 19, 48, "W", "Walla Walla", WA
31, 32, 59, "N", 97, 8, 23, "W", "Waco", TX
38, 40, 48, "N", 87, 31, 47, "W", "Vincennes", IN
28, 48, 35, "N", 97, 0, 36, "W", "Victoria", TX
32, 20, 59, "N", 90, 52, 47, "W", "Vicksburg", MS
49, 16, 12, "N", 123, 7, 12, "W", "Vancouver", BC
46, 55, 11, "N", 98, 0, 36, "W", "Valley City", ND
30, 49, 47, "N", 83, 16, 47, "W", "Valdosta", GA
43, 6, 36, "N", 75, 13, 48, "W", "Utica", NY
39, 54, 0, "N", 79, 43, 48, "W", "Uniontown", PA
32, 20, 59, "N", 95, 18, 0, "W", "Tyler", TX
42, 33, 36, "N", 114, 28, 12, "W", "Twin Falls", ID
33, 12, 35, "N", 87, 34, 11, "W", "Tuscaloosa", AL
34, 15, 35, "N", 88, 42, 35, "W", "Tupelo", MS
36, 9, 35, "N", 95, 54, 36, "W", "Tulsa", OK
32, 13, 12, "N", 110, 58, 12, "W", "Tucson", AZ
37, 10, 11, "N", 104, 30, 36, "W", "Trinidad", CO
40, 13, 47, "N", 74, 46, 11, "W", "Trenton", NJ
44, 45, 35, "N", 85, 37, 47, "W", "Traverse City", MI
43, 39, 0, "N", 79, 22, 47, "W", "Toronto", ON
39, 2, 59, "N", 95, 40, 11, "W", "Topeka", KS
41, 39, 0, "N", 83, 32, 24, "W", "Toledo", OH
33, 25, 48, "N", 94, 3, 0, "W", "Texarkana", TX
39, 28, 12, "N", 87, 24, 36, "W", "Terre Haute", IN
27, 57, 0, "N", 82, 26, 59, "W", "Tampa", FL
30, 27, 0, "N", 84, 16, 47, "W", "Tallahassee", FL
47, 14, 24, "N", 122, 25, 48, "W", "Tacoma", WA
43, 2, 59, "N", 76, 9, 0, "W", "Syracuse", NY
32, 35, 59, "N", 82, 20, 23, "W", "Swainsboro", GA
33, 55, 11, "N", 80, 20, 59, "W", "Sumter", SC
40, 59, 24, "N", 75, 11, 24, "W", "Stroudsburg", PA
37, 57, 35, "N", 121, 17, 24, "W", "Stockton", CA
44, 31, 12, "N", 89, 34, 11, "W", "Stevens Point", WI
40, 21, 36, "N", 80, 37, 12, "W", "Steubenville", OH
40, 37, 11, "N", 103, 13, 12, "W", "Sterling", CO
38, 9, 0, "N", 79, 4, 11, "W", "Staunton", VA
39, 55, 11, "N", 83, 48, 35, "W", "Springfield", OH
37, 13, 12, "N", 93, 17, 24, "W", "Springfield", MO
42, 5, 59, "N", 72, 35, 23, "W", "Springfield", MA
39, 47, 59, "N", 89, 39, 0, "W", "Springfield", IL
47, 40, 11, "N", 117, 24, 36, "W", "Spokane", WA
41, 40, 48, "N", 86, 15, 0, "W", "South Bend", IN
43, 32, 24, "N", 96, 43, 48, "W", "Sioux Falls", SD
42, 29, 24, "N", 96, 23, 23, "W", "Sioux City", IA
32, 30, 35, "N", 93, 45, 0, "W", "Shreveport", LA
33, 38, 23, "N", 96, 36, 36, "W", "Sherman", TX
44, 47, 59, "N", 106, 57, 35, "W", "Sheridan", WY
35, 13, 47, "N", 96, 40, 48, "W", "Seminole", OK
32, 25, 11, "N", 87, 1, 11, "W", "Selma", AL
38, 42, 35, "N", 93, 13, 48, "W", "Sedalia", MO
47, 35, 59, "N", 122, 19, 48, "W", "Seattle", WA
41, 24, 35, "N", 75, 40, 11, "W", "Scranton", PA
41, 52, 11, "N", 103, 39, 36, "W", "Scottsbluff", NB
42, 49, 11, "N", 73, 56, 59, "W", "Schenectady", NY
32, 4, 48, "N", 81, 5, 23, "W", "Savannah", GA
46, 29, 24, "N", 84, 20, 59, "W", "Sault Sainte Marie", MI
27, 20, 24, "N", 82, 31, 47, "W", "Sarasota", FL
38, 26, 23, "N", 122, 43, 12, "W", "Santa Rosa", CA
35, 40, 48, "N", 105, 56, 59, "W", "Santa Fe", NM
34, 25, 11, "N", 119, 41, 59, "W", "Santa Barbara", CA
33, 45, 35, "N", 117, 52, 12, "W", "Santa Ana", CA
37, 20, 24, "N", 121, 52, 47, "W", "San Jose", CA
37, 46, 47, "N", 122, 25, 11, "W", "San Francisco", CA
41, 27, 0, "N", 82, 42, 35, "W", "Sandusky", OH
32, 42, 35, "N", 117, 9, 0, "W", "San Diego", CA
34, 6, 36, "N", 117, 18, 35, "W", "San Bernardino", CA
29, 25, 12, "N", 98, 30, 0, "W", "San Antonio", TX
31, 27, 35, "N", 100, 26, 24, "W", "San Angelo", TX
40, 45, 35, "N", 111, 52, 47, "W", "Salt Lake City", UT
38, 22, 11, "N", 75, 35, 59, "W", "Salisbury", MD
36, 40, 11, "N", 121, 39, 0, "W", "Salinas", CA
38, 50, 24, "N", 97, 36, 36, "W", "Salina", KS
38, 31, 47, "N", 106, 0, 0, "W", "Salida", CO
44, 56, 23, "N", 123, 1, 47, "W", "Salem", OR
44, 57, 0, "N", 93, 5, 59, "W", "Saint Paul", MN
38, 37, 11, "N", 90, 11, 24, "W", "Saint Louis", MO
39, 46, 12, "N", 94, 50, 23, "W", "Saint Joseph", MO
42, 5, 59, "N", 86, 28, 48, "W", "Saint Joseph", MI
44, 25, 11, "N", 72, 1, 11, "W", "Saint Johnsbury", VT
45, 34, 11, "N", 94, 10, 11, "W", "Saint Cloud", MN
29, 53, 23, "N", 81, 19, 11, "W", "Saint Augustine", FL
43, 25, 48, "N", 83, 56, 24, "W", "Saginaw", MI
38, 35, 24, "N", 121, 29, 23, "W", "Sacramento", CA
43, 36, 36, "N", 72, 58, 12, "W", "Rutland", VT
33, 24, 0, "N", 104, 31, 47, "W", "Roswell", NM
35, 56, 23, "N", 77, 48, 0, "W", "Rocky Mount", NC
41, 35, 24, "N", 109, 13, 48, "W", "Rock Springs", WY
42, 16, 12, "N", 89, 5, 59, "W", "Rockford", IL
43, 9, 35, "N", 77, 36, 36, "W", "Rochester", NY
44, 1, 12, "N", 92, 27, 35, "W", "Rochester", MN
37, 16, 12, "N", 79, 56, 24, "W", "Roanoke", VA
37, 32, 24, "N", 77, 26, 59, "W", "Richmond", VA
39, 49, 48, "N", 84, 53, 23, "W", "Richmond", IN
38, 46, 12, "N", 112, 5, 23, "W", "Richfield", UT
45, 38, 23, "N", 89, 25, 11, "W", "Rhinelander", WI
39, 31, 12, "N", 119, 48, 35, "W", "Reno", NV
50, 25, 11, "N", 104, 39, 0, "W", "Regina", SA
40, 10, 48, "N", 122, 14, 23, "W", "Red Bluff", CA
40, 19, 48, "N", 75, 55, 48, "W", "Reading", PA
41, 9, 35, "N", 81, 14, 23, "W", "Ravenna", OH
1 LatD LatM LatS NS LonD LonM LonS EW City State
2 41 5 59 N 80 39 0 W Youngstown OH
3 42 52 48 N 97 23 23 W Yankton SD
4 46 35 59 N 120 30 36 W Yakima WA
5 42 16 12 N 71 48 0 W Worcester MA
6 43 37 48 N 89 46 11 W Wisconsin Dells WI
7 36 5 59 N 80 15 0 W Winston-Salem NC
8 49 52 48 N 97 9 0 W Winnipeg MB
9 39 11 23 N 78 9 36 W Winchester VA
10 34 14 24 N 77 55 11 W Wilmington NC
11 39 45 0 N 75 33 0 W Wilmington DE
12 48 9 0 N 103 37 12 W Williston ND
13 41 15 0 N 77 0 0 W Williamsport PA
14 37 40 48 N 82 16 47 W Williamson WV
15 33 54 0 N 98 29 23 W Wichita Falls TX
16 37 41 23 N 97 20 23 W Wichita KS
17 40 4 11 N 80 43 12 W Wheeling WV
18 26 43 11 N 80 3 0 W West Palm Beach FL
19 47 25 11 N 120 19 11 W Wenatchee WA
20 41 25 11 N 122 23 23 W Weed CA
21 31 13 11 N 82 20 59 W Waycross GA
22 44 57 35 N 89 38 23 W Wausau WI
23 42 21 36 N 87 49 48 W Waukegan IL
24 44 54 0 N 97 6 36 W Watertown SD
25 43 58 47 N 75 55 11 W Watertown NY
26 42 30 0 N 92 20 23 W Waterloo IA
27 41 32 59 N 73 3 0 W Waterbury CT
28 38 53 23 N 77 1 47 W Washington DC
29 41 50 59 N 79 8 23 W Warren PA
30 46 4 11 N 118 19 48 W Walla Walla WA
31 31 32 59 N 97 8 23 W Waco TX
32 38 40 48 N 87 31 47 W Vincennes IN
33 28 48 35 N 97 0 36 W Victoria TX
34 32 20 59 N 90 52 47 W Vicksburg MS
35 49 16 12 N 123 7 12 W Vancouver BC
36 46 55 11 N 98 0 36 W Valley City ND
37 30 49 47 N 83 16 47 W Valdosta GA
38 43 6 36 N 75 13 48 W Utica NY
39 39 54 0 N 79 43 48 W Uniontown PA
40 32 20 59 N 95 18 0 W Tyler TX
41 42 33 36 N 114 28 12 W Twin Falls ID
42 33 12 35 N 87 34 11 W Tuscaloosa AL
43 34 15 35 N 88 42 35 W Tupelo MS
44 36 9 35 N 95 54 36 W Tulsa OK
45 32 13 12 N 110 58 12 W Tucson AZ
46 37 10 11 N 104 30 36 W Trinidad CO
47 40 13 47 N 74 46 11 W Trenton NJ
48 44 45 35 N 85 37 47 W Traverse City MI
49 43 39 0 N 79 22 47 W Toronto ON
50 39 2 59 N 95 40 11 W Topeka KS
51 41 39 0 N 83 32 24 W Toledo OH
52 33 25 48 N 94 3 0 W Texarkana TX
53 39 28 12 N 87 24 36 W Terre Haute IN
54 27 57 0 N 82 26 59 W Tampa FL
55 30 27 0 N 84 16 47 W Tallahassee FL
56 47 14 24 N 122 25 48 W Tacoma WA
57 43 2 59 N 76 9 0 W Syracuse NY
58 32 35 59 N 82 20 23 W Swainsboro GA
59 33 55 11 N 80 20 59 W Sumter SC
60 40 59 24 N 75 11 24 W Stroudsburg PA
61 37 57 35 N 121 17 24 W Stockton CA
62 44 31 12 N 89 34 11 W Stevens Point WI
63 40 21 36 N 80 37 12 W Steubenville OH
64 40 37 11 N 103 13 12 W Sterling CO
65 38 9 0 N 79 4 11 W Staunton VA
66 39 55 11 N 83 48 35 W Springfield OH
67 37 13 12 N 93 17 24 W Springfield MO
68 42 5 59 N 72 35 23 W Springfield MA
69 39 47 59 N 89 39 0 W Springfield IL
70 47 40 11 N 117 24 36 W Spokane WA
71 41 40 48 N 86 15 0 W South Bend IN
72 43 32 24 N 96 43 48 W Sioux Falls SD
73 42 29 24 N 96 23 23 W Sioux City IA
74 32 30 35 N 93 45 0 W Shreveport LA
75 33 38 23 N 96 36 36 W Sherman TX
76 44 47 59 N 106 57 35 W Sheridan WY
77 35 13 47 N 96 40 48 W Seminole OK
78 32 25 11 N 87 1 11 W Selma AL
79 38 42 35 N 93 13 48 W Sedalia MO
80 47 35 59 N 122 19 48 W Seattle WA
81 41 24 35 N 75 40 11 W Scranton PA
82 41 52 11 N 103 39 36 W Scottsbluff NB
83 42 49 11 N 73 56 59 W Schenectady NY
84 32 4 48 N 81 5 23 W Savannah GA
85 46 29 24 N 84 20 59 W Sault Sainte Marie MI
86 27 20 24 N 82 31 47 W Sarasota FL
87 38 26 23 N 122 43 12 W Santa Rosa CA
88 35 40 48 N 105 56 59 W Santa Fe NM
89 34 25 11 N 119 41 59 W Santa Barbara CA
90 33 45 35 N 117 52 12 W Santa Ana CA
91 37 20 24 N 121 52 47 W San Jose CA
92 37 46 47 N 122 25 11 W San Francisco CA
93 41 27 0 N 82 42 35 W Sandusky OH
94 32 42 35 N 117 9 0 W San Diego CA
95 34 6 36 N 117 18 35 W San Bernardino CA
96 29 25 12 N 98 30 0 W San Antonio TX
97 31 27 35 N 100 26 24 W San Angelo TX
98 40 45 35 N 111 52 47 W Salt Lake City UT
99 38 22 11 N 75 35 59 W Salisbury MD
100 36 40 11 N 121 39 0 W Salinas CA
101 38 50 24 N 97 36 36 W Salina KS
102 38 31 47 N 106 0 0 W Salida CO
103 44 56 23 N 123 1 47 W Salem OR
104 44 57 0 N 93 5 59 W Saint Paul MN
105 38 37 11 N 90 11 24 W Saint Louis MO
106 39 46 12 N 94 50 23 W Saint Joseph MO
107 42 5 59 N 86 28 48 W Saint Joseph MI
108 44 25 11 N 72 1 11 W Saint Johnsbury VT
109 45 34 11 N 94 10 11 W Saint Cloud MN
110 29 53 23 N 81 19 11 W Saint Augustine FL
111 43 25 48 N 83 56 24 W Saginaw MI
112 38 35 24 N 121 29 23 W Sacramento CA
113 43 36 36 N 72 58 12 W Rutland VT
114 33 24 0 N 104 31 47 W Roswell NM
115 35 56 23 N 77 48 0 W Rocky Mount NC
116 41 35 24 N 109 13 48 W Rock Springs WY
117 42 16 12 N 89 5 59 W Rockford IL
118 43 9 35 N 77 36 36 W Rochester NY
119 44 1 12 N 92 27 35 W Rochester MN
120 37 16 12 N 79 56 24 W Roanoke VA
121 37 32 24 N 77 26 59 W Richmond VA
122 39 49 48 N 84 53 23 W Richmond IN
123 38 46 12 N 112 5 23 W Richfield UT
124 45 38 23 N 89 25 11 W Rhinelander WI
125 39 31 12 N 119 48 35 W Reno NV
126 50 25 11 N 104 39 0 W Regina SA
127 40 10 48 N 122 14 23 W Red Bluff CA
128 40 19 48 N 75 55 48 W Reading PA
129 41 9 35 N 81 14 23 W Ravenna OH

24
media/testdata/resource.ics vendored Normal file
View file

@ -0,0 +1,24 @@
BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//ZContent.net//Zap Calendar 1.0//EN
CALSCALE:GREGORIAN
METHOD:PUBLISH
BEGIN:VEVENT
SUMMARY:Abraham Lincoln
UID:c7614cff-3549-4a00-9152-d25cc1fe077d
SEQUENCE:0
STATUS:CONFIRMED
TRANSP:TRANSPARENT
RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=2;BYMONTHDAY=12
DTSTART:20080212
DTEND:20080213
DTSTAMP:20150421T141403
CATEGORIES:U.S. Presidents,Civil War People
LOCATION:Hodgenville\, Kentucky
GEO:37.5739497;-85.7399606
DESCRIPTION:Born February 12\, 1809\nSixteenth President (1861-1865)\n\n\n
\nhttp://AmericanHistoryCalendar.com
URL:http://americanhistorycalendar.com/peoplecalendar/1,328-abraham-lincol
n
END:VEVENT
END:VCALENDAR

BIN
media/testdata/resource.jpg vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 114 KiB

3
media/testdata/resource.js vendored Normal file
View file

@ -0,0 +1,3 @@
function foo() {
return "foo";
}

14
media/testdata/resource.json vendored Normal file
View file

@ -0,0 +1,14 @@
{
"firstName": "Joe",
"lastName": "Jackson",
"gender": "male",
"age": 28,
"address": {
"streetAddress": "101",
"city": "San Diego",
"state": "CA"
},
"phoneNumbers": [
{ "type": "home", "number": "7349282382" }
]
}

BIN
media/testdata/resource.png vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

20
media/testdata/resource.rss vendored Normal file
View file

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8" ?>
<rss version="2.0">
<channel>
<title>W3Schools Home Page</title>
<link>https://www.w3schools.com</link>
<description>Free web building tutorials</description>
<item>
<title>RSS Tutorial</title>
<link>https://www.w3schools.com/xml/xml_rss.asp</link>
<description>New RSS tutorial on W3Schools</description>
</item>
<item>
<title>XML Tutorial</title>
<link>https://www.w3schools.com/xml</link>
<description>New XML tutorial on W3Schools</description>
</item>
</channel>
</rss>

6
media/testdata/resource.sass vendored Normal file
View file

@ -0,0 +1,6 @@
$font-stack: Helvetica, sans-serif
$primary-color: #333
body
font: 100% $font-stack
color: $primary-color

7
media/testdata/resource.scss vendored Normal file
View file

@ -0,0 +1,7 @@
$font-stack: Helvetica, sans-serif;
$primary-color: #333;
body {
font: 100% $font-stack;
color: $primary-color;
}

5
media/testdata/resource.svg vendored Normal file
View file

@ -0,0 +1,5 @@
<svg height="100" width="100">
<circle cx="50" cy="50" r="40" stroke="black" stroke-width="3" fill="red" />
Sorry, your browser does not support inline SVG.
</svg>

After

Width:  |  Height:  |  Size: 172 B

BIN
media/testdata/resource.ttf vendored Normal file

Binary file not shown.

BIN
media/testdata/resource.webp vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 58 KiB

7
media/testdata/resource.xml vendored Normal file
View file

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<note>
<to>Tove</to>
<from>Jani</from>
<heading>Reminder</heading>
<body>Don't forget me this weekend!</body>
</note>

View file

@ -20,6 +20,7 @@ import (
"strings" "strings"
"github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/media"
"github.com/pkg/errors" "github.com/pkg/errors"
@ -45,6 +46,15 @@ var (
".webp": WEBP, ".webp": WEBP,
} }
imageFormatsBySubType = map[string]Format{
media.JPEGType.SubType: JPEG,
media.PNGType.SubType: PNG,
media.TIFFType.SubType: TIFF,
media.BMPType.SubType: BMP,
media.GIFType.SubType: GIF,
media.WEBPType.SubType: WEBP,
}
// Add or increment if changes to an image format's processing requires // Add or increment if changes to an image format's processing requires
// re-generation. // re-generation.
imageFormatsVersions = map[Format]int{ imageFormatsVersions = map[Format]int{
@ -102,6 +112,11 @@ func ImageFormatFromExt(ext string) (Format, bool) {
return f, found return f, found
} }
func ImageFormatFromMediaSubType(sub string) (Format, bool) {
f, found := imageFormatsBySubType[sub]
return f, found
}
const ( const (
defaultJPEGQuality = 75 defaultJPEGQuality = 75
defaultResampleFilter = "box" defaultResampleFilter = "box"

View file

@ -66,6 +66,9 @@ func (*Filters) Text(text string, options ...interface{}) gift.Filter {
case "linespacing": case "linespacing":
tf.linespacing = cast.ToInt(v) tf.linespacing = cast.ToInt(v)
case "font": case "font":
if err, ok := v.(error); ok {
panic(fmt.Sprintf("invalid font source: %s", err))
}
fontSource, ok1 := v.(hugio.ReadSeekCloserProvider) fontSource, ok1 := v.(hugio.ReadSeekCloserProvider)
identifier, ok2 := v.(resource.Identifier) identifier, ok2 := v.(resource.Identifier)

View file

@ -36,6 +36,7 @@ func TestCreatePlaceholders(t *testing.T) {
"Suffixes": "pre_foo.Suffixes_post", "Suffixes": "pre_foo.Suffixes_post",
"Delimiter": "pre_foo.Delimiter_post", "Delimiter": "pre_foo.Delimiter_post",
"FirstSuffix": "pre_foo.FirstSuffix_post", "FirstSuffix": "pre_foo.FirstSuffix_post",
"IsText": "pre_foo.IsText_post",
"String": "pre_foo.String_post", "String": "pre_foo.String_post",
"Type": "pre_foo.Type_post", "Type": "pre_foo.Type_post",
"MainType": "pre_foo.MainType_post", "MainType": "pre_foo.MainType_post",

View file

@ -69,6 +69,9 @@ type ResourceSourceDescriptor struct {
Fs afero.Fs Fs afero.Fs
// Set when its known up front, else it's resolved from the target filename.
MediaType media.Type
// The relative target filename without any language code. // The relative target filename without any language code.
RelTargetFilename string RelTargetFilename string

View file

@ -29,6 +29,7 @@ import (
"github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/resources/resource"
"github.com/mitchellh/mapstructure" "github.com/mitchellh/mapstructure"
@ -99,7 +100,7 @@ func (c *Client) FromRemote(uri string, optionsm map[string]interface{}) (resour
body, err := ioutil.ReadAll(res.Body) body, err := ioutil.ReadAll(res.Body)
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "failed to read remote resource %s", uri) return nil, errors.Wrapf(err, "failed to read remote resource %q", uri)
} }
filename := path.Base(rURL.Path) filename := path.Base(rURL.Path)
@ -109,33 +110,30 @@ func (c *Client) FromRemote(uri string, optionsm map[string]interface{}) (resour
} }
} }
var extension string var extensionHint string
if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 { if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
extension = arr[0] extensionHint = arr[0]
} }
// If extension was not determined by header, look for a file extention // Look for a file extention
if extension == "" { if extensionHint == "" {
if ext := path.Ext(filename); ext != "" { if ext := path.Ext(filename); ext != "" {
extension = ext extensionHint = ext
} }
} }
// If extension was not determined by header or file extention, try using content itself // Now resolve the media type primarily using the content.
if extension == "" { mediaType := media.FromContent(c.rs.MediaTypes, extensionHint, body)
if ct := http.DetectContentType(body); ct != "application/octet-stream" { if mediaType.IsZero() {
if ct == "image/jpeg" { return nil, errors.Errorf("failed to resolve media type for remote resource %q", uri)
extension = ".jpg"
} else if arr, _ := mime.ExtensionsByType(ct); arr != nil {
extension = arr[0]
}
}
} }
resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + extension resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + mediaType.FirstSuffix.FullSuffix
return c.rs.New( return c.rs.New(
resources.ResourceSourceDescriptor{ resources.ResourceSourceDescriptor{
MediaType: mediaType,
LazyPublish: true, LazyPublish: true,
OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil

View file

@ -272,8 +272,14 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso
fd.RelTargetFilename = sourceFilename fd.RelTargetFilename = sourceFilename
} }
mimeType := fd.MediaType
if mimeType.IsZero() {
ext := strings.ToLower(filepath.Ext(fd.RelTargetFilename)) ext := strings.ToLower(filepath.Ext(fd.RelTargetFilename))
mimeType, suffixInfo, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, ".")) var (
found bool
suffixInfo media.SuffixInfo
)
mimeType, suffixInfo, found = r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, "."))
// TODO(bep) we need to handle these ambiguous types better, but in this context // TODO(bep) we need to handle these ambiguous types better, but in this context
// we most likely want the application/xml type. // we most likely want the application/xml type.
if suffixInfo.Suffix == "xml" && mimeType.SubType == "rss" { if suffixInfo.Suffix == "xml" && mimeType.SubType == "rss" {
@ -289,6 +295,7 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso
mimeType, _ = media.FromStringAndExt(mimeStr, ext) mimeType, _ = media.FromStringAndExt(mimeStr, ext)
} }
} }
}
gr := r.newGenericResourceWithBase( gr := r.newGenericResourceWithBase(
sourceFs, sourceFs,
@ -301,7 +308,7 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso
mimeType) mimeType)
if mimeType.MainType == "image" { if mimeType.MainType == "image" {
imgFormat, ok := images.ImageFormatFromExt(ext) imgFormat, ok := images.ImageFormatFromMediaSubType(mimeType.SubType)
if ok { if ok {
ir := &imageResource{ ir := &imageResource{
Image: images.NewImage(imgFormat, r.imaging, nil, gr), Image: images.NewImage(imgFormat, r.imaging, nil, gr),

View file

@ -110,30 +110,21 @@ func (ns *Namespace) getscssClientDartSass() (*dartsass.Client, error) {
// Get locates the filename given in Hugo's assets filesystem and // Get locates the filename given in Hugo's assets filesystem and
// creates a Resource object that can be used for // creates a Resource object that can be used for
// further transformations. // further transformations.
func (ns *Namespace) Get(filename interface{}) resource.Resource { func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
get := func(args ...interface{}) (resource.Resource, error) {
filenamestr, err := cast.ToStringE(filename) filenamestr, err := cast.ToStringE(filename)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return ns.createClient.Get(filepath.Clean(filenamestr)) return ns.createClient.Get(filepath.Clean(filenamestr))
}
r, err := get(filename)
if err != nil {
// This allows the client to reason about the .Err in the template.
// This is not as relevant for local resources as remotes, but
// it makes this method work the same way as resources.GetRemote.
return resources.NewErrorResource(errors.Wrap(err, "error calling resources.Get"))
}
return r
} }
// GetRemote gets the URL (via HTTP(s)) in the first argument in args and creates Resource object that can be used for // GetRemote gets the URL (via HTTP(s)) in the first argument in args and creates Resource object that can be used for
// further transformations. // further transformations.
// //
// A second argument may be provided with an option map. // A second argument may be provided with an option map.
//
// Note: This method does not return any error as a second argument,
// for any error situations the error can be checked in .Err.
func (ns *Namespace) GetRemote(args ...interface{}) resource.Resource { func (ns *Namespace) GetRemote(args ...interface{}) resource.Resource {
get := func(args ...interface{}) (resource.Resource, error) { get := func(args ...interface{}) (resource.Resource, error) {
if len(args) < 1 { if len(args) < 1 {

View file

@ -37,7 +37,6 @@ import (
"github.com/gohugoio/hugo/tpl/internal" "github.com/gohugoio/hugo/tpl/internal"
"github.com/gohugoio/hugo/tpl/partials" "github.com/gohugoio/hugo/tpl/partials"
"github.com/spf13/afero" "github.com/spf13/afero"
) )
var logger = loggers.NewErrorLogger() var logger = loggers.NewErrorLogger()

View file

@ -95,6 +95,10 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) {
return nil, errors.Errorf("type %T not supported", data) return nil, errors.Errorf("type %T not supported", data)
} }
if dataStr == "" {
return nil, errors.New("no data to transform")
}
key := helpers.MD5String(dataStr) key := helpers.MD5String(dataStr)
return ns.cache.GetOrCreate(key, func() (interface{}, error) { return ns.cache.GetOrCreate(key, func() (interface{}, error) {