diff --git a/commands/commandeer.go b/commands/commandeer.go
index d5d2740bf..4ca0c4be9 100644
--- a/commands/commandeer.go
+++ b/commands/commandeer.go
@@ -37,23 +37,27 @@ import (
 	"github.com/gohugoio/hugo/langs"
 )
 
-type commandeer struct {
+type commandeerHugoState struct {
 	*deps.DepsCfg
+	hugo     *hugolib.HugoSites
+	fsCreate sync.Once
+}
 
-	hugo *hugolib.HugoSites
+type commandeer struct {
+	*commandeerHugoState
+
+	// We need to reuse this on server rebuilds.
+	destinationFs afero.Fs
 
 	h    *hugoBuilderCommon
 	ftch flagsToConfigHandler
 
 	visitedURLs *types.EvictingStringQueue
 
-	// We watch these for changes.
-	configFiles []string
-
 	doWithCommandeer func(c *commandeer) error
 
-	// We can do this only once.
-	fsCreate sync.Once
+	// We watch these for changes.
+	configFiles []string
 
 	// Used in cases where we get flooded with events in server mode.
 	debounce func(f func())
@@ -73,6 +77,7 @@ func (c *commandeer) Set(key string, value interface{}) {
 }
 
 func (c *commandeer) initFs(fs *hugofs.Fs) error {
+	c.destinationFs = fs.Destination
 	c.DepsCfg.Fs = fs
 
 	return nil
@@ -89,11 +94,12 @@ func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f fla
 	}
 
 	c := &commandeer{
-		h:                h,
-		ftch:             f,
-		doWithCommandeer: doWithCommandeer,
-		visitedURLs:      types.NewEvictingStringQueue(10),
-		debounce:         rebuildDebouncer,
+		h:                   h,
+		ftch:                f,
+		commandeerHugoState: &commandeerHugoState{},
+		doWithCommandeer:    doWithCommandeer,
+		visitedURLs:         types.NewEvictingStringQueue(10),
+		debounce:            rebuildDebouncer,
 	}
 
 	return c, c.loadConfig(mustHaveConfigFile, running)
@@ -188,8 +194,11 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 	c.fsCreate.Do(func() {
 		fs := hugofs.NewFrom(sourceFs, config)
 
-		// Hugo writes the output to memory instead of the disk.
-		if createMemFs {
+		if c.destinationFs != nil {
+			// Need to reuse the destination on server rebuilds.
+			fs.Destination = c.destinationFs
+		} else if createMemFs {
+			// Hugo writes the output to memory instead of the disk.
 			fs.Destination = new(afero.MemMapFs)
 		}
 
diff --git a/commands/hugo.go b/commands/hugo.go
index 5c87bd96e..2b847ec95 100644
--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -596,14 +596,6 @@ func (c *commandeer) getDirList() ([]string, error) {
 	return a, nil
 }
 
-func (c *commandeer) recreateAndBuildSites(watching bool) (err error) {
-	defer c.timeTrack(time.Now(), "Total")
-	if !c.h.quiet {
-		c.Logger.FEEDBACK.Println("Started building sites ...")
-	}
-	return c.hugo.Build(hugolib.BuildCfg{CreateSitesFromConfig: true})
-}
-
 func (c *commandeer) resetAndBuildSites() (err error) {
 	if !c.h.quiet {
 		c.Logger.FEEDBACK.Println("Started building sites ...")
@@ -637,9 +629,10 @@ func (c *commandeer) rebuildSites(events []fsnotify.Event) error {
 }
 
 func (c *commandeer) fullRebuild() {
+	c.commandeerHugoState = &commandeerHugoState{}
 	if err := c.loadConfig(true, true); err != nil {
 		jww.ERROR.Println("Failed to reload config:", err)
-	} else if err := c.recreateAndBuildSites(true); err != nil {
+	} else if err := c.buildSites(); err != nil {
 		jww.ERROR.Println(err)
 	} else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") {
 		livereload.ForceRefresh()
diff --git a/commands/server.go b/commands/server.go
index 8089b0ade..fde881f6f 100644
--- a/commands/server.go
+++ b/commands/server.go
@@ -298,7 +298,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro
 		}
 	}
 
-	httpFs := afero.NewHttpFs(f.c.Fs.Destination)
+	httpFs := afero.NewHttpFs(f.c.destinationFs)
 	fs := filesOnlyFs{httpFs.Dir(absPublishDir)}
 
 	doLiveReload := !f.s.buildWatch && !f.c.Cfg.GetBool("disableLiveReload")