Fix HEAD method in resources.GetRemote

Fixes #10604
This commit is contained in:
Bjørn Erik Pedersen 2023-01-16 11:05:28 +01:00
parent b5d485060f
commit f13531e608
4 changed files with 135 additions and 43 deletions

View file

@ -117,7 +117,7 @@ func FromContent(types Types, extensionHints []string, content []byte) Type {
// FromStringAndExt creates a Type from a MIME string and a given extension. // FromStringAndExt creates a Type from a MIME string and a given extension.
func FromStringAndExt(t, ext string) (Type, error) { func FromStringAndExt(t, ext string) (Type, error) {
tp, err := fromString(t) tp, err := FromString(t)
if err != nil { if err != nil {
return tp, err return tp, err
} }
@ -129,7 +129,7 @@ func FromStringAndExt(t, ext string) (Type, error) {
// FromString creates a new Type given a type string on the form MainType/SubType and // FromString creates a new Type given a type string on the form MainType/SubType and
// an optional suffix, e.g. "text/html" or "text/html+html". // an optional suffix, e.g. "text/html" or "text/html+html".
func fromString(t string) (Type, error) { func FromString(t string) (Type, error) {
t = strings.ToLower(t) t = strings.ToLower(t)
parts := strings.Split(t, "/") parts := strings.Split(t, "/")
if len(parts) != 2 { if len(parts) != 2 {
@ -470,7 +470,7 @@ func DecodeTypes(mms ...map[string]any) (Types, error) {
mediaType, found := mmm[k] mediaType, found := mmm[k]
if !found { if !found {
var err error var err error
mediaType, err = fromString(k) mediaType, err = FromString(k)
if err != nil { if err != nil {
return m, err return m, err
} }

View file

@ -132,22 +132,22 @@ func TestGetFirstBySuffix(t *testing.T) {
func TestFromTypeString(t *testing.T) { func TestFromTypeString(t *testing.T) {
c := qt.New(t) c := qt.New(t)
f, err := fromString("text/html") f, err := FromString("text/html")
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
c.Assert(f.Type(), qt.Equals, HTMLType.Type()) c.Assert(f.Type(), qt.Equals, HTMLType.Type())
f, err = fromString("application/custom") f, err = FromString("application/custom")
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
c.Assert(f, qt.Equals, Type{MainType: "application", SubType: "custom", mimeSuffix: ""}) c.Assert(f, qt.Equals, Type{MainType: "application", SubType: "custom", mimeSuffix: ""})
f, err = fromString("application/custom+sfx") f, err = FromString("application/custom+sfx")
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
c.Assert(f, qt.Equals, Type{MainType: "application", SubType: "custom", mimeSuffix: "sfx"}) c.Assert(f, qt.Equals, Type{MainType: "application", SubType: "custom", mimeSuffix: "sfx"})
_, err = fromString("noslash") _, err = FromString("noslash")
c.Assert(err, qt.Not(qt.IsNil)) c.Assert(err, qt.Not(qt.IsNil))
f, err = fromString("text/xml; charset=utf-8") f, err = FromString("text/xml; charset=utf-8")
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
c.Assert(f, qt.Equals, Type{MainType: "text", SubType: "xml", mimeSuffix: ""}) c.Assert(f, qt.Equals, Type{MainType: "text", SubType: "xml", mimeSuffix: ""})

View file

@ -0,0 +1,56 @@
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package create_test
import (
"testing"
"github.com/gohugoio/hugo/hugolib"
)
func TestGetResourceHead(t *testing.T) {
files := `
-- config.toml --
[security]
[security.http]
methods = ['(?i)GET|POST|HEAD']
urls = ['.*gohugo\.io.*']
-- layouts/index.html --
{{ $url := "https://gohugo.io/img/hugo.png" }}
{{ $opts := dict "method" "head" }}
{{ with resources.GetRemote $url $opts }}
{{ with .Err }}
{{ errorf "Unable to get remote resource: %s" . }}
{{ else }}
Head Content: {{ .Content }}.
{{ end }}
{{ else }}
{{ errorf "Unable to get remote resource: %s" $url }}
{{ end }}
`
b := hugolib.NewIntegrationTestBuilder(
hugolib.IntegrationTestConfig{
T: t,
TxtarString: files,
},
)
b.Build()
b.AssertFileContent("public/index.html", "Head Content: .")
}

View file

@ -45,7 +45,29 @@ type HTTPError struct {
Body string Body string
} }
func toHTTPError(err error, res *http.Response) *HTTPError { func responseToData(res *http.Response, readBody bool) map[string]any {
var body []byte
if readBody {
body, _ = ioutil.ReadAll(res.Body)
}
m := map[string]any{
"StatusCode": res.StatusCode,
"Status": res.Status,
"TransferEncoding": res.TransferEncoding,
"ContentLength": res.ContentLength,
"ContentType": res.Header.Get("Content-Type"),
}
if readBody {
m["Body"] = string(body)
}
return m
}
func toHTTPError(err error, res *http.Response, readBody bool) *HTTPError {
if err == nil { if err == nil {
panic("err is nil") panic("err is nil")
} }
@ -56,19 +78,9 @@ func toHTTPError(err error, res *http.Response) *HTTPError {
} }
} }
var body []byte
body, _ = ioutil.ReadAll(res.Body)
return &HTTPError{ return &HTTPError{
error: err, error: err,
Data: map[string]any{ Data: responseToData(res, readBody),
"StatusCode": res.StatusCode,
"Status": res.Status,
"Body": string(body),
"TransferEncoding": res.TransferEncoding,
"ContentLength": res.ContentLength,
"ContentType": res.Header.Get("Content-Type"),
},
} }
} }
@ -80,6 +92,12 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
return nil, fmt.Errorf("failed to parse URL for resource %s: %w", uri, err) return nil, fmt.Errorf("failed to parse URL for resource %s: %w", uri, err)
} }
method := "GET"
if s, ok := maps.LookupEqualFold(optionsm, "method"); ok {
method = strings.ToUpper(s.(string))
}
isHeadMethod := method == "HEAD"
resourceID := calculateResourceID(uri, optionsm) resourceID := calculateResourceID(uri, optionsm)
_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) { _, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
@ -100,15 +118,16 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer res.Body.Close()
httpResponse, err := httputil.DumpResponse(res, true) httpResponse, err := httputil.DumpResponse(res, true)
if err != nil { if err != nil {
return nil, toHTTPError(err, res) return nil, toHTTPError(err, res, !isHeadMethod)
} }
if res.StatusCode != http.StatusNotFound { if res.StatusCode != http.StatusNotFound {
if res.StatusCode < 200 || res.StatusCode > 299 { if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res) return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res, !isHeadMethod)
} }
} }
@ -124,15 +143,24 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer res.Body.Close()
if res.StatusCode == http.StatusNotFound { if res.StatusCode == http.StatusNotFound {
// Not found. This matches how looksup for local resources work. // Not found. This matches how looksup for local resources work.
return nil, nil return nil, nil
} }
body, err := ioutil.ReadAll(res.Body) var (
if err != nil { body []byte
return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err) mediaType media.Type
)
// A response to a HEAD method should not have a body. If it has one anyway, that body must be ignored.
// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD
if !isHeadMethod && res.Body != nil {
body, err = ioutil.ReadAll(res.Body)
if err != nil {
return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err)
}
} }
filename := path.Base(rURL.Path) filename := path.Base(rURL.Path)
@ -142,30 +170,38 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
} }
} }
var extensionHints []string
contentType := res.Header.Get("Content-Type") contentType := res.Header.Get("Content-Type")
// mime.ExtensionsByType gives a long list of extensions for text/plain, if isHeadMethod {
// just use ".txt". // We have no body to work with, so we need to use the Content-Type header.
if strings.HasPrefix(contentType, "text/plain") { mediaType, _ = media.FromString(contentType)
extensionHints = []string{".txt"}
} else { } else {
exts, _ := mime.ExtensionsByType(contentType)
if exts != nil { var extensionHints []string
extensionHints = exts
// mime.ExtensionsByType gives a long list of extensions for text/plain,
// just use ".txt".
if strings.HasPrefix(contentType, "text/plain") {
extensionHints = []string{".txt"}
} else {
exts, _ := mime.ExtensionsByType(contentType)
if exts != nil {
extensionHints = exts
}
} }
// Look for a file extension. If it's .txt, look for a more specific.
if extensionHints == nil || extensionHints[0] == ".txt" {
if ext := path.Ext(filename); ext != "" {
extensionHints = []string{ext}
}
}
// Now resolve the media type primarily using the content.
mediaType = media.FromContent(c.rs.MediaTypes, extensionHints, body)
} }
// Look for a file extension. If it's .txt, look for a more specific.
if extensionHints == nil || extensionHints[0] == ".txt" {
if ext := path.Ext(filename); ext != "" {
extensionHints = []string{ext}
}
}
// Now resolve the media type primarily using the content.
mediaType := media.FromContent(c.rs.MediaTypes, extensionHints, body)
if mediaType.IsZero() { if mediaType.IsZero() {
return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri) return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri)
} }