Refactored the pusher for cleaner code and to easily plug the poller in

src/pusher/common/common.go

@@ -1,7 +1,6 @@
 package common
 
 import (
-	"io/ioutil"
 	"strings"
 
 	"config"
@@ -11,94 +10,61 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// PrepareForPush reads the file containing the JSON representation of a
-// dashboard, checks if the dashboard is set to be ignored, and if not appends
-// its content to a map, which will be later iterated over to push the contents
-// it contains to the Grafana API.
-// Returns an error if there was an issue reading the file or checking if the
-// dashboard it represents is to be ignored.
-func PrepareForPush(
-	filename string, filesToPush *map[string][]byte, cfg *config.Config,
+func FilterIgnored(
+	filesToPush *map[string][]byte, cfg *config.Config,
 ) (err error) {
-	// Don't set versions.json to be pushed
-	if strings.HasSuffix(filename, "versions.json") {
-		return
-	}
-
-	// Read the file's content
-	fileContent, err := ioutil.ReadFile(filename)
-	if err != nil {
-		return
-	}
-
-	// Check if dashboard is ignored
-	ignored, err := isIgnored(fileContent, cfg)
-	if err != nil {
-		return
-	}
+	for filename, content := range *filesToPush {
+		// Don't set versions.json to be pushed
+		if strings.HasSuffix(filename, "versions.json") {
+			delete(*filesToPush, filename)
+			continue
+		}
 
-	// Append to the list of contents to push to Grafana
-	if !ignored {
-		logrus.WithFields(logrus.Fields{
-			"filename": filename,
-		}).Info("Preparing file to be pushed to Grafana")
+		// Check if dashboard is ignored
+		ignored, err := isIgnored(content, cfg)
+		if err != nil {
+			return err
+		}
 
-		(*filesToPush)[filename] = fileContent
+		if ignored {
+			delete(*filesToPush, filename)
+		}
 	}
 
 	return
 }
 
-func PushFiles(filesToPush map[string][]byte, client *grafana.Client) {
+func PushFiles(filenames []string, contents map[string][]byte, client *grafana.Client) {
 	// Push all files to the Grafana API
-	for fileToPush, fileContent := range filesToPush {
-		if err := client.CreateOrUpdateDashboard(fileContent); err != nil {
+	for _, filename := range filenames {
+		if err := client.CreateOrUpdateDashboard(contents[filename]); err != nil {
 			logrus.WithFields(logrus.Fields{
 				"error":    err,
-				"filename": fileToPush,
+				"filename": filename,
 			}).Error("Failed to push the file to Grafana")
 		}
 	}
 }
 
-// DeleteDashboard reads the dashboard described in a given file and, if the file
-// isn't set to be ignored, delete the corresponding dashboard from Grafana.
-// Returns an error if there was an issue reading the file's content, checking
-// if the dashboard is to be ignored, computing its slug or deleting it from
-// Grafana.
-func DeleteDashboard(
-	filename string, client *grafana.Client, cfg *config.Config,
-) (err error) {
-	// Don't delete versions.json
-	if strings.HasSuffix(filename, "versions.json") {
-		return
-	}
-
-	// Read the file's content
-	fileContent, err := ioutil.ReadFile(filename)
-	if err != nil {
-		return
-	}
-
-	// Check if dashboard is ignored
-	ignored, err := isIgnored(fileContent, cfg)
-	if err != nil {
-		return
-	}
-
-	if !ignored {
+func DeleteDashboards(filenames []string, contents map[string][]byte, client *grafana.Client) {
+	for _, filename := range filenames {
 		// Retrieve dashboard slug because we need it in the deletion request.
-		var slug string
-		slug, err = helpers.GetDashboardSlug(fileContent)
+		slug, err := helpers.GetDashboardSlug(contents[filename])
 		if err != nil {
-			return
+			logrus.WithFields(logrus.Fields{
+				"error":    err,
+				"filename": filename,
+			}).Error("Failed to compute the dahsboard's slug")
 		}
 
-		// Delete the dashboard
-		err = client.DeleteDashboard(slug)
+		if err := client.DeleteDashboard(slug); err != nil {
+			logrus.WithFields(logrus.Fields{
+				"error":    err,
+				"filename": filename,
+				"slug":     slug,
+			}).Error("Failed to remove the dashboard from Grafana")
+		}
 	}
-
-	return
 }
 
 // isIgnored checks whether the file must be ignored, by checking if there's an

src/pusher/poller/poller.go

@@ -45,12 +45,15 @@ func poller(
 		return
 	}
 
+	filesContents, err := repo.GetFilesContentsAtCommit(latestCommit)
+	if err != nil {
+		return
+	}
+
 	previousCommit := latestCommit
+	previousFilesContents := filesContents
 
 	for {
-		addedOrModified := make([]string, 0)
-		removed := make([]string, 0)
-
 		if err = repo.Sync(true); err != nil {
 			return
 		}
@@ -61,56 +64,48 @@ func poller(
 		}
 
 		if previousCommit.Hash.String() != latestCommit.Hash.String() {
-			lineCounts, err := git.GetFilesLineCountsAtCommit(previousCommit)
+			logrus.WithFields(logrus.Fields{
+				"previous_hash": previousCommit.Hash.String(),
+				"new_hash":      latestCommit.Hash.String(),
+			}).Info("New commit(s) detected")
+
+			filesContents, err = repo.GetFilesContentsAtCommit(latestCommit)
 			if err != nil {
 				return err
 			}
 
-			deltas, err := repo.LineCountsDeltasIgnoreManagerCommits(previousCommit, latestCommit)
+			modified, removed, err := repo.GetModifiedAndRemovedFiles(previousCommit, latestCommit)
 			if err != nil {
 				return err
 			}
 
-			for file, delta := range deltas {
-				if delta == 0 {
-					continue
-				}
-
-				if delta > 0 {
-					addedOrModified = append(addedOrModified, file)
-				} else if delta < 0 {
-					if -delta < lineCounts[file] {
-						addedOrModified = append(addedOrModified, file)
-					} else {
-						removed = append(removed, file)
-					}
-				}
-			}
-		}
+			mergedContents := mergeContents(modified, removed, filesContents, previousFilesContents)
+			common.PushFiles(modified, mergedContents, client)
 
-		filesToPush := make(map[string][]byte)
-		for _, filename := range addedOrModified {
-			if err = common.PrepareForPush(filename, &filesToPush, cfg); err != nil {
-				return err
-			}
-		}
-
-		common.PushFiles(filesToPush, client)
-
-		if delRemoved {
-			for _, removedFile := range removed {
-				if err = common.DeleteDashboard(
-					removedFile, client, cfg,
-				); err != nil {
-					logrus.WithFields(logrus.Fields{
-						"error":    err,
-						"filename": removedFile,
-					}).Error("Failed to delete the dashboard")
-				}
+			if delRemoved {
+				common.DeleteDashboards(removed, mergedContents, client)
 			}
 		}
 
 		previousCommit = latestCommit
+		previousFilesContents = filesContents
 		time.Sleep(time.Duration(cfg.Pusher.Config.Interval) * time.Second)
 	}
 }
+
+func mergeContents(
+	modified []string, removed []string,
+	filesContents map[string][]byte, previousFilesContents map[string][]byte,
+) (merged map[string][]byte) {
+	merged = make(map[string][]byte)
+
+	for _, modifiedFile := range modified {
+		merged[modifiedFile] = filesContents[modifiedFile]
+	}
+
+	for _, removedFile := range removed {
+		merged[removedFile] = previousFilesContents[removedFile]
+	}
+
+	return
+}

src/pusher/webhook/webhook.go

@@ -1,6 +1,9 @@
 package webhook
 
 import (
+	"io/ioutil"
+	"path/filepath"
+
 	"config"
 	"git"
 	"grafana"
@@ -49,6 +52,13 @@ func Setup(conf *config.Config, client *grafana.Client, delRemoved bool) (err er
 func HandlePush(payload interface{}, header webhooks.Header) {
 	var err error
 
+	var (
+		added    = make([]string, 0)
+		modified = make([]string, 0)
+		removed  = make([]string, 0)
+		contents = make(map[string][]byte)
+	)
+
 	// Process the payload using the right structure
 	pl := payload.(gitlab.PushEventPayload)
 
@@ -57,25 +67,6 @@ func HandlePush(payload interface{}, header webhooks.Header) {
 		return
 	}
 
-	// Clone or pull the repository
-	if err = repo.Sync(false); err != nil {
-		logrus.WithFields(logrus.Fields{
-			"error":      err,
-			"repo":       cfg.Git.User + "@" + cfg.Git.URL,
-			"clone_path": cfg.Git.ClonePath,
-		}).Error("Failed to synchronise the Git repository with the remote")
-
-		return
-	}
-
-	// Files to push and their contents are stored in a map before being pushed
-	// to the Grafana API. We don't push them in the loop iterating over commits
-	// because, in the case a file is successively updated by two commits pushed
-	// at the same time, it would push the same file several time, which isn't
-	// an optimised behaviour.
-	filesToPush := make(map[string][]byte)
-
-	// Iterate over the commits descriptions from the payload
 	for _, commit := range pl.Commits {
 		// We don't want to process commits made by the puller
 		if commit.Author.Email == cfg.Git.CommitsAuthor.Email {
@@ -88,56 +79,52 @@ func HandlePush(payload interface{}, header webhooks.Header) {
 			continue
 		}
 
-		// Set all added files to be pushed, except the ones describing a
-		// dashboard which name starts with a the prefix specified in the
-		// configuration file.
 		for _, addedFile := range commit.Added {
-			if err = common.PrepareForPush(
-				addedFile, &filesToPush, cfg,
-			); err != nil {
-				logrus.WithFields(logrus.Fields{
-					"error":    err,
-					"filename": addedFile,
-				}).Error("Failed to prepare file for push")
-
-				continue
-			}
+			added = append(added, addedFile)
 		}
 
-		// Set all modified files to be pushed, except the ones describing a
-		// dashboard which name starts with a the prefix specified in the
-		// configuration file.
 		for _, modifiedFile := range commit.Modified {
-			if err = common.PrepareForPush(
-				modifiedFile, &filesToPush, cfg,
-			); err != nil {
-				logrus.WithFields(logrus.Fields{
-					"error":    err,
-					"filename": modifiedFile,
-				}).Error("Failed to prepare file for push")
-
-				continue
-			}
+			modified = append(modified, modifiedFile)
 		}
 
-		// If a file describing a dashboard gets removed from the Git repository,
-		// delete the corresponding dashboard on Grafana, but only if the user
-		// mentionned they want to do so with the correct command line flag.
-		if deleteRemoved {
-			for _, removedFile := range commit.Removed {
-				if err = common.DeleteDashboard(
-					removedFile, grafanaClient, cfg,
-				); err != nil {
-					logrus.WithFields(logrus.Fields{
-						"error":    err,
-						"filename": removedFile,
-					}).Error("Failed to delete the dashboard")
-				}
-			}
+		for _, removedFile := range commit.Removed {
+			removed = append(removed, removedFile)
 		}
 	}
 
-	common.PushFiles(filesToPush, grafanaClient)
+	if err = getFilesContents(removed, &contents, cfg); err != nil {
+		return
+	}
+
+	// Clone or pull the repository
+	if err = repo.Sync(false); err != nil {
+		logrus.WithFields(logrus.Fields{
+			"error":      err,
+			"repo":       cfg.Git.User + "@" + cfg.Git.URL,
+			"clone_path": cfg.Git.ClonePath,
+		}).Error("Failed to synchronise the Git repository with the remote")
+
+		return
+	}
+
+	if err = getFilesContents(added, &contents, cfg); err != nil {
+		return
+	}
+
+	if err = getFilesContents(modified, &contents, cfg); err != nil {
+		return
+	}
+
+	if err = common.FilterIgnored(&contents, cfg); err != nil {
+		return
+	}
+
+	common.PushFiles(added, contents, grafanaClient)
+	common.PushFiles(modified, contents, grafanaClient)
+
+	if deleteRemoved {
+		common.DeleteDashboards(removed, contents, grafanaClient)
+	}
 
 	// Grafana will auto-update the version number after we pushed the new
 	// dashboards, so we use the puller mechanic to pull the updated numbers and
@@ -150,3 +137,20 @@ func HandlePush(payload interface{}, header webhooks.Header) {
 		}).Error("Call to puller returned an error")
 	}
 }
+
+func getFilesContents(
+	filenames []string, contents *map[string][]byte, cfg *config.Config,
+) (err error) {
+	for _, filename := range filenames {
+		// Read the file's content
+		filePath := filepath.Join(cfg.Git.ClonePath, filename)
+		fileContent, err := ioutil.ReadFile(filePath)
+		if err != nil {
+			return err
+		}
+
+		(*contents)[filename] = fileContent
+	}
+
+	return
+}