[WIP] Feature/populate

.editorconfig

@@ -14,3 +14,6 @@ indent_size = 8
 [*.{html,json,js,jsx,scss}]
 indent_style = space
 indent_size = 2
+
+[{Makefile,**.mk}]
+indent_style = tab

.gitignore

@@ -1,13 +1,17 @@
 .env
-vendor/*
-starred.json
-public/node_modules/*
-node_modules
+*.db
+*.index
+*.json
+*.py
+build.sh
 dist
-mix-manifest.json
 fonts
+kleng
+mix-manifest.json
+my.db
+node_modules
+public/node_modules/*
+starred.json
 tmp
 toolkit
-kleng
-*.db
-*.index
+vendor/*

Makefile

@@ -0,0 +1,21 @@
+deps:
+	@echo "Installing dependencies"
+	@glide install
+
+build:
+	@echo "Building Kleng"
+	@go build -x -v -race -o kleng
+
+run:
+	@echo "Running kleng with HTTP mode"
+	./kleng -M http
+
+populate:
+	@echo "Running kleng with Populate mode"
+	./kleng -M populate
+
+clean:
+	@rm -f kleng
+	@go clean all
+
+.PHONY: deps build populate clean run

core/db.go

@@ -0,0 +1,21 @@
+package core
+
+import (
+	"time"
+
+	"github.com/boltdb/bolt"
+)
+
+// OpenDB open bolt db
+func OpenDB() (*bolt.DB, error) {
+
+	if DB == nil {
+		db, err := bolt.Open("my.db", 0600, &bolt.Options{Timeout: 1 * time.Second})
+		if err != nil {
+			return new(bolt.DB), err
+		}
+		DB = db
+	}
+
+	return DB, nil
+}

core/gh.go

@@ -47,11 +47,3 @@ func (g GithubClient) ListStarred(page int) ([]*github.StarredRepository, error)
 
 	return stared, err
 }
-
-func (g GithubClient) PopulateStarred() error {
-	// for {
-	/// todo
-	// }
-
-	return nil
-}

core/global.go

@@ -0,0 +1,10 @@
+package core
+
+import (
+	"github.com/boltdb/bolt"
+)
+
+var (
+	// DB a global instance of boltDB
+	DB *bolt.DB
+)

core/populate.go

@@ -0,0 +1,149 @@
+package core
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+
+	"strconv"
+
+	"github.com/boltdb/bolt"
+	"github.com/google/go-github/github"
+	"github.com/redite/kleng/utils"
+)
+
+type requestControl struct {
+	starred []*github.StarredRepository
+	err     []error
+	wg      sync.WaitGroup
+	mu      *sync.RWMutex
+}
+
+// Populate perform call to user's starred repo on github and store the result to bolt database
+func Populate() error {
+	return firstRun()
+}
+
+func firstRun() error {
+
+	rc := &requestControl{
+		mu: new(sync.RWMutex),
+	}
+
+	c := utils.NewGithubClient()
+	pageCount, _ := getStarredPageCount(c.Username)
+
+	// diparallel aja biar cepet.
+	for page := 0; page <= pageCount; page++ {
+		rc.wg.Add(1)
+		go func(idx int, stack *requestControl) {
+			defer stack.wg.Done()
+
+			res, err := c.ListStarred(idx)
+			if err != nil {
+				stack.mu.Lock()
+				stack.err = append(stack.err, err)
+				stack.mu.Unlock()
+				return
+			}
+
+			// ga boleh rebutan, nanti dimarahin mama.
+			stack.mu.Lock()
+			stack.starred = append(stack.starred, res...)
+			stack.mu.Unlock()
+		}(page, rc)
+	}
+
+	// tungguin sampe goroutine kelar semua.
+	rc.wg.Wait()
+
+	if len(rc.err) > 0 {
+		fmt.Printf("got %d error", len(rc.err))
+		fmt.Println(rc.err)
+	}
+
+	db, err := OpenDB()
+	if err != nil {
+		err = fmt.Errorf("failed opening DB: %s", err.Error())
+		return err
+	}
+
+	defer db.Close()
+
+	err = db.Update(func(tx *bolt.Tx) error {
+		bucket, err := tx.CreateBucketIfNotExists([]byte("kleng"))
+		if err != nil {
+			return err
+		}
+
+		j, err := json.Marshal(rc.starred)
+		if err != nil {
+			return fmt.Errorf("cannot marshal data: %s", err.Error())
+		}
+
+		err = bucket.Put([]byte("starred_repo"), j)
+		return err
+	})
+
+	err = db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte("kleng"))
+		if bucket == nil {
+			return fmt.Errorf("Bucket %q not found!", []byte("kleng"))
+		}
+
+		val := bucket.Get([]byte("starred_repo"))
+		fmt.Println(string(val))
+
+		return nil
+	})
+
+	// j, err := json.Marshal(rc.starred)
+	// err = ioutil.WriteFile("./new-starred-1.json", j, 0644)
+	// if err != nil {
+	// return err
+	// }
+
+	return nil
+}
+
+func populationUpdate() error {
+	return nil
+}
+
+func getStarredPageCount(username string) (count int, err error) {
+	client := http.DefaultClient
+	resp, err := client.Get(
+		fmt.Sprintf("%s/%s/%s", "https://api.github.com/users", username, "starred"),
+	)
+	if err != nil {
+		return 0, err
+	}
+	defer resp.Body.Close()
+
+	links := resp.Header.Get("Link")
+	if len(links) > 0 {
+		lastpage := strings.Replace(
+			strings.Replace(
+				strings.Split(links, ";")[1],
+				"rel=\"next\", <",
+				"",
+				-1,
+			),
+			">",
+			"",
+			-1,
+		)
+
+		lasturl, err := url.Parse(lastpage)
+		if err != nil {
+			return count, err
+		}
+
+		count, err = strconv.Atoi(lasturl.Query().Get("page"))
+	}
+
+	return count, err
+}

main.go

@@ -1,19 +1,39 @@
 package main
 
 import (
+	"flag"
 	"fmt"
+	"log"
 	"os"
 
 	"github.com/redite/kleng/config"
+	"github.com/redite/kleng/core"
 	"github.com/redite/kleng/server"
 )
 
+var (
+	mode = flag.String("M", "", "run with `mode`. available mode: 'populate', 'http'")
+)
+
 func main() {
 	conf, err := config.LoadConfig()
 	if err != nil {
 		fmt.Printf("Got error: %s", err.Error())
 		os.Exit(-1)
 	}
 
-	server.RunServer(conf)
+	switch *mode {
+	case "http":
+		log.Fatal(server.RunServer(conf))
+		break
+	case "populate":
+		err = core.Populate()
+		if err != nil {
+			log.Printf("Error Populating: %s\n", err.Error())
+		}
+		break
+	default:
+		flag.Usage()
+	}
+
 }