diff --git a/config/configLoader.go b/config/configLoader.go
index f2265dc6a..b8aa3fda3 100644
--- a/config/configLoader.go
+++ b/config/configLoader.go
@@ -14,6 +14,9 @@
 package config
 
 import (
+	"path/filepath"
+	"strings"
+
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/parser/metadecoders"
 	"github.com/spf13/afero"
@@ -21,9 +24,23 @@ import (
 )
 
 var (
-	ValidConfigFileExtensions = []string{"toml", "yaml", "yml", "json"}
+	ValidConfigFileExtensions                    = []string{"toml", "yaml", "yml", "json"}
+	validConfigFileExtensionsMap map[string]bool = make(map[string]bool)
 )
 
+func init() {
+	for _, ext := range ValidConfigFileExtensions {
+		validConfigFileExtensionsMap[ext] = true
+	}
+}
+
+// IsValidConfigFilename returns whether filename is one of the supported
+// config formats in Hugo.
+func IsValidConfigFilename(filename string) bool {
+	ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(filename), "."))
+	return validConfigFileExtensionsMap[ext]
+}
+
 // FromConfigString creates a config from the given YAML, JSON or TOML config. This is useful in tests.
 func FromConfigString(config, configType string) (Provider, error) {
 	v := newViper()
diff --git a/config/configLoader_test.go b/config/configLoader_test.go
new file mode 100644
index 000000000..06a00df3b
--- /dev/null
+++ b/config/configLoader_test.go
@@ -0,0 +1,34 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsValidConfigFileName(t *testing.T) {
+	assert := require.New(t)
+
+	for _, ext := range ValidConfigFileExtensions {
+		filename := "config." + ext
+		assert.True(IsValidConfigFilename(filename), ext)
+		assert.True(IsValidConfigFilename(strings.ToUpper(filename)))
+	}
+
+	assert.False(IsValidConfigFilename(""))
+	assert.False(IsValidConfigFilename("config.toml.swp"))
+}
diff --git a/hugolib/config.go b/hugolib/config.go
index 5b56333df..6a1de32be 100644
--- a/hugolib/config.go
+++ b/hugolib/config.go
@@ -283,6 +283,10 @@ func (l configLoader) loadConfigFromConfigDir(v *viper.Viper) ([]string, error)
 				return nil
 			}
 
+			if !config.IsValidConfigFilename(path) {
+				return nil
+			}
+
 			name := helpers.Filename(filepath.Base(path))
 
 			item, err := metadecoders.Default.UnmarshalFileToMap(sourceFs, path)
diff --git a/hugolib/configdir_test.go b/hugolib/configdir_test.go
index 80fcda61f..c1afbb14e 100644
--- a/hugolib/configdir_test.go
+++ b/hugolib/configdir_test.go
@@ -97,7 +97,9 @@ p3 = "p3params_no_production"
 	fb = fb.WithWorkingDir("config/development")
 
 	// This is set in all the config.toml variants above, but this will win.
-	fb.Add("config.toml", `paginatePath = "pag_development"`)
+	fb.Add("config.TOML", `paginatePath = "pag_development"`)
+	// Issue #5646
+	fb.Add("config.toml.swp", `p3 = "paginatePath = "nono"`)
 
 	fb.Add("params.no.toml", `p3 = "p3params_no_development"`)
 	fb.Add("params.toml", `p3 = "p3params_development"`)