chore(config): organize log directory in current/archive style

cmd/batch_clone_build/build.go

@@ -98,7 +98,11 @@ func buildGrp(cfg *config.Artifact, wg *sync.WaitGroup, resChan chan CreateDBRes
 }
 
 func buildDirSetup(cfg *config.Artifact) (*os.File, *os.File) {
-	logdir := cfg.PassLogDir("build")
+	const passname = "build"
+	if _, err := config.ArchiveCurrentIfExist(cfg, passname); err != nil {
+		log.Fatalf("Failed to archive current log dir: %v", err)
+	}
+	logdir := cfg.PassLogDir(passname)
 	// Create output files
 	csvFilePath := filepath.Join(logdir, "repoTimes.csv")
 	csvFile := utils.CreateFile(csvFilePath)

cmd/batch_clone_build/clone.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"log"
 	"os"
 	"path/filepath"
 	"sync"
@@ -26,6 +27,11 @@ func dirSetup(cfg *config.Artifact) {
 }
 
 func batchClone(cfg *config.Artifact) {
+	const passname = "clone"
+	if _, err := config.ArchiveCurrentIfExist(cfg, passname); err != nil {
+		log.Fatalf("Failed to archive current log dir: %v", err)
+	}
+	logdir := cfg.PassLogDir(passname)
 	type cloneStatus struct {
 		fullname string
 		err      error // nil means success
@@ -98,7 +104,6 @@ func batchClone(cfg *config.Artifact) {
 	}
 
 	if len(fails) != 0 {
-		logdir := cfg.PassLogDir("clone")
 		failFile := utils.CreateFile(filepath.Join(logdir, "fail.log"))
 		defer failFile.Close()
 

cmd/codeql_qdriver/main.go

@@ -61,12 +61,18 @@ func main() {
 		runtime.GOMAXPROCS(cfg.ParallelCore)
 	}
 	if !onlyDecode {
+		if _, err := config.ArchiveCurrentIfExist(&cfg, "query"); err != nil {
+			log.Fatalf("Failed to archive current log dir: %v", err)
+		}
 		for grpi, grp := range cfg.QueryGrps {
 			fmt.Printf("Grp %d: Executing queries\n", grpi)
 			queriesExec(grp)
 		}
 	}
 	fmt.Println("Decoding results")
+	if _, err := config.ArchiveCurrentIfExist(&cfg, "decode"); err != nil {
+		log.Fatalf("Failed to archive current log dir: %v", err)
+	}
 	decodeResults(targetDecodeFmt)
 	if doCollect && targetDecodeFmt == "csv" {
 		fmt.Println("Collecting CSVs")

config/artifact.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"time"
@@ -58,20 +59,70 @@ type ExternalGenGroup struct {
 
 var Nowstr string = time.Now().Local().Format("0102-150405")
 
-var logDirMap map[string]string = make(map[string]string)
+var logDirCache map[string]string = make(map[string]string) // pass to logDir cache
 
+// logMetaTy is the metadata of a log directory
+type logMetaTy struct {
+	Pass string `json:"pass"`
+	Now  string `json:"now"`
+}
+
+const logMetaFileName string = "logMeta.json"
+
+func (l *logMetaTy) dump(file string) error {
+	bs, err := json.Marshal(l)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(file, bs, 0644)
+}
+func loadLogMeta(file string) (res logMetaTy, err error) {
+	var bs []byte
+	bs, err = os.ReadFile(file)
+	if err != nil {
+		return
+	}
+	err = json.Unmarshal(bs, &res)
+	return
+}
+
+// PassLogDir returns the log directory for the pass, logRoot/pass/current
 func (art *Artifact) PassLogDir(pass string) string {
-	if dir, ok := logDirMap[pass]; ok {
+	if dir, ok := logDirCache[pass]; ok {
 		return dir
 	}
-	dir := filepath.Join(art.LogRoot, pass, Nowstr)
+	dir := filepath.Join(art.LogRoot, pass, "current")
 	if _, err := os.Stat(dir); err != nil {
 		utils.MkdirAll(dir)
 	}
-	logDirMap[pass] = dir
+	logDirCache[pass] = dir
+	logMeta := logMetaTy{
+		Pass: pass,
+		Now:  Nowstr,
+	}
+	logMeta.dump(filepath.Join(dir, logMetaFileName))
 	return dir
 }
 
+// ArchiveCurrentIfExist archives the current log directory if it exists according to logMeta.json
+func ArchiveCurrentIfExist(art *Artifact, pass string) (exist bool, err error) {
+	srcdir := filepath.Join(art.LogRoot, pass, "current")
+	if _, err = os.Stat(srcdir); err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		} else {
+			return true, err
+		}
+	}
+	meta, err := loadLogMeta(filepath.Join(srcdir, logMetaFileName))
+	if err != nil {
+		return true, err
+	}
+	tgtdir := filepath.Join(art.LogRoot, pass, "archive", meta.Now)
+	utils.MkdirAll(filepath.Dir(tgtdir))
+	return true, os.Rename(srcdir, tgtdir)
+}
+
 type reposType int
 
 const (