diff --git a/.editorconfig b/.editorconfig index cfbc947..1d66e34 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,3 +14,6 @@ indent_size = 8 [*.{html,json,js,jsx,scss}] indent_style = space indent_size = 2 + +[{Makefile,**.mk}] +indent_style = tab diff --git a/.gitignore b/.gitignore index 25fed0c..803fa70 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,17 @@ .env -vendor/* -starred.json -public/node_modules/* -node_modules +*.db +*.index +*.json +*.py +build.sh dist -mix-manifest.json fonts +kleng +mix-manifest.json +my.db +node_modules +public/node_modules/* +starred.json tmp toolkit -kleng -*.db -*.index \ No newline at end of file +vendor/* \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cf843d4 --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +deps: + @echo "Installing dependencies" + @glide install + +build: + @echo "Building Kleng" + @go build -x -v -race -o kleng + +run: + @echo "Running kleng with HTTP mode" + ./kleng -M http + +populate: + @echo "Running kleng with Populate mode" + ./kleng -M populate + +clean: + @rm -f kleng + @go clean all + +.PHONY: deps build populate clean run \ No newline at end of file diff --git a/core/db.go b/core/db.go new file mode 100644 index 0000000..06d6b40 --- /dev/null +++ b/core/db.go @@ -0,0 +1,21 @@ +package core + +import ( + "time" + + "github.com/boltdb/bolt" +) + +// OpenDB open bolt db +func OpenDB() (*bolt.DB, error) { + + if DB == nil { + db, err := bolt.Open("my.db", 0600, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return new(bolt.DB), err + } + DB = db + } + + return DB, nil +} diff --git a/core/gh.go b/core/gh.go index e85c052..4bcd326 100644 --- a/core/gh.go +++ b/core/gh.go @@ -47,11 +47,3 @@ func (g GithubClient) ListStarred(page int) ([]*github.StarredRepository, error) return stared, err } - -func (g GithubClient) PopulateStarred() error { - // for { - /// todo - // } - - return nil -} diff --git a/core/global.go b/core/global.go new file mode 100644 index 0000000..980b8b0 --- /dev/null +++ b/core/global.go @@ -0,0 +1,10 @@ +package core + +import ( + "github.com/boltdb/bolt" +) + +var ( + // DB a global instance of boltDB + DB *bolt.DB +) diff --git a/core/populate.go b/core/populate.go new file mode 100644 index 0000000..f71c3cc --- /dev/null +++ b/core/populate.go @@ -0,0 +1,149 @@ +package core + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "sync" + + "strconv" + + "github.com/boltdb/bolt" + "github.com/google/go-github/github" + "github.com/redite/kleng/utils" +) + +type requestControl struct { + starred []*github.StarredRepository + err []error + wg sync.WaitGroup + mu *sync.RWMutex +} + +// Populate perform call to user's starred repo on github and store the result to bolt database +func Populate() error { + return firstRun() +} + +func firstRun() error { + + rc := &requestControl{ + mu: new(sync.RWMutex), + } + + c := utils.NewGithubClient() + pageCount, _ := getStarredPageCount(c.Username) + + // diparallel aja biar cepet. + for page := 0; page <= pageCount; page++ { + rc.wg.Add(1) + go func(idx int, stack *requestControl) { + defer stack.wg.Done() + + res, err := c.ListStarred(idx) + if err != nil { + stack.mu.Lock() + stack.err = append(stack.err, err) + stack.mu.Unlock() + return + } + + // ga boleh rebutan, nanti dimarahin mama. + stack.mu.Lock() + stack.starred = append(stack.starred, res...) + stack.mu.Unlock() + }(page, rc) + } + + // tungguin sampe goroutine kelar semua. + rc.wg.Wait() + + if len(rc.err) > 0 { + fmt.Printf("got %d error", len(rc.err)) + fmt.Println(rc.err) + } + + db, err := OpenDB() + if err != nil { + err = fmt.Errorf("failed opening DB: %s", err.Error()) + return err + } + + defer db.Close() + + err = db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte("kleng")) + if err != nil { + return err + } + + j, err := json.Marshal(rc.starred) + if err != nil { + return fmt.Errorf("cannot marshal data: %s", err.Error()) + } + + err = bucket.Put([]byte("starred_repo"), j) + return err + }) + + err = db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte("kleng")) + if bucket == nil { + return fmt.Errorf("Bucket %q not found!", []byte("kleng")) + } + + val := bucket.Get([]byte("starred_repo")) + fmt.Println(string(val)) + + return nil + }) + + // j, err := json.Marshal(rc.starred) + // err = ioutil.WriteFile("./new-starred-1.json", j, 0644) + // if err != nil { + // return err + // } + + return nil +} + +func populationUpdate() error { + return nil +} + +func getStarredPageCount(username string) (count int, err error) { + client := http.DefaultClient + resp, err := client.Get( + fmt.Sprintf("%s/%s/%s", "https://api.github.com/users", username, "starred"), + ) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + links := resp.Header.Get("Link") + if len(links) > 0 { + lastpage := strings.Replace( + strings.Replace( + strings.Split(links, ";")[1], + "rel=\"next\", <", + "", + -1, + ), + ">", + "", + -1, + ) + + lasturl, err := url.Parse(lastpage) + if err != nil { + return count, err + } + + count, err = strconv.Atoi(lasturl.Query().Get("page")) + } + + return count, err +} diff --git a/main.go b/main.go index 5126488..a50d619 100644 --- a/main.go +++ b/main.go @@ -1,13 +1,20 @@ package main import ( + "flag" "fmt" + "log" "os" "github.com/redite/kleng/config" + "github.com/redite/kleng/core" "github.com/redite/kleng/server" ) +var ( + mode = flag.String("M", "", "run with `mode`. available mode: 'populate', 'http'") +) + func main() { conf, err := config.LoadConfig() if err != nil { @@ -15,5 +22,18 @@ func main() { os.Exit(-1) } - server.RunServer(conf) + switch *mode { + case "http": + log.Fatal(server.RunServer(conf)) + break + case "populate": + err = core.Populate() + if err != nil { + log.Printf("Error Populating: %s\n", err.Error()) + } + break + default: + flag.Usage() + } + }