From f4ae6c738e066866fbee71762d4a0fd011c14d81 Mon Sep 17 00:00:00 2001 From: widnyana Date: Mon, 20 Mar 2017 19:34:23 +0700 Subject: [PATCH 1/5] update dev stuff --- .editorconfig | 3 +++ .gitignore | 5 +++++ Makefile | 19 +++++++++++++++++++ build.sh | 9 --------- package.json | 7 ++++--- 5 files changed, 31 insertions(+), 12 deletions(-) create mode 100644 Makefile delete mode 100755 build.sh diff --git a/.editorconfig b/.editorconfig index 61ac2b0..99e2aa8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,3 +14,6 @@ indent_size = 8 [*.{html,json,js}] indent_style = space indent_size = 2 + +[{Makefile,**.mk}] +indent_style = tab diff --git a/.gitignore b/.gitignore index d2614e3..26bdea1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,8 @@ node_modules dist mix-manifest.json fonts +kleng +my.db +*.py +*.json +build.sh diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0807d27 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +build: + @echo "Building Kleng" + @go build -x -v -race -o kleng + +run: + @echo "Running kleng with HTTP mode" + @make build + ./kleng -M http + +populate: + @echo "Running kleng with Populate mode" + @make build + ./kleng -M populate + +clean: + @rm -f kleng + @go clean all + +.PHONY: deps build diff --git a/build.sh b/build.sh deleted file mode 100755 index 02479d1..0000000 --- a/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -echo -e "Compiling resounden.jsx" -cd ./public/js -babel --presets "react" -o resounden.js resounden.jsx - -echo -e "Running server..." -cd ../../ -go run main.go \ No newline at end of file diff --git a/package.json b/package.json index 8e4935b..23d07ed 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,13 @@ { "scripts": { - "dev": "node node_modules/cross-env/bin/cross-env.js NODE_ENV=development node_modules/webpack/bin/webpack.js --progress --hide-modules --config=node_modules/laravel-mix/setup/webpack.config.js", - "watch": "node node_modules/cross-env/bin/cross-env.js NODE_ENV=development node_modules/webpack/bin/webpack.js --watch --progress --hide-modules --config=node_modules/laravel-mix/setup/webpack.config.js" + "dev": "node node_modules/cross-env/dist/bin/cross-env.js NODE_ENV=development node_modules/webpack/bin/webpack.js --progress --hide-modules --config=node_modules/laravel-mix/setup/webpack.config.js", + "watch": "node node_modules/cross-env/dist/bin/cross-env.js NODE_ENV=development node_modules/webpack/bin/webpack.js --watch --progress --hide-modules --config=node_modules/laravel-mix/setup/webpack.config.js" }, "devDependencies": { "babel-preset-react": "^6.23.0", "bootstrap-sass": "^3.3.7", - "laravel-mix": "^0.8.8" + "laravel-mix": "^0.8.8", + "cross-env": "^3.2.4" }, "dependencies": { "jquery": "^3.1.1", From 67dff98be6cf6c35296af195043a37a9623b2cb4 Mon Sep 17 00:00:00 2001 From: widnyana Date: Mon, 20 Mar 2017 19:34:56 +0700 Subject: [PATCH 2/5] add db initiator and populate functionality --- core/db.go | 21 +++++++ core/global.go | 10 ++++ core/populate.go | 149 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 core/db.go create mode 100644 core/global.go create mode 100644 core/populate.go diff --git a/core/db.go b/core/db.go new file mode 100644 index 0000000..06d6b40 --- /dev/null +++ b/core/db.go @@ -0,0 +1,21 @@ +package core + +import ( + "time" + + "github.com/boltdb/bolt" +) + +// OpenDB open bolt db +func OpenDB() (*bolt.DB, error) { + + if DB == nil { + db, err := bolt.Open("my.db", 0600, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return new(bolt.DB), err + } + DB = db + } + + return DB, nil +} diff --git a/core/global.go b/core/global.go new file mode 100644 index 0000000..980b8b0 --- /dev/null +++ b/core/global.go @@ -0,0 +1,10 @@ +package core + +import ( + "github.com/boltdb/bolt" +) + +var ( + // DB a global instance of boltDB + DB *bolt.DB +) diff --git a/core/populate.go b/core/populate.go new file mode 100644 index 0000000..f71c3cc --- /dev/null +++ b/core/populate.go @@ -0,0 +1,149 @@ +package core + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "sync" + + "strconv" + + "github.com/boltdb/bolt" + "github.com/google/go-github/github" + "github.com/redite/kleng/utils" +) + +type requestControl struct { + starred []*github.StarredRepository + err []error + wg sync.WaitGroup + mu *sync.RWMutex +} + +// Populate perform call to user's starred repo on github and store the result to bolt database +func Populate() error { + return firstRun() +} + +func firstRun() error { + + rc := &requestControl{ + mu: new(sync.RWMutex), + } + + c := utils.NewGithubClient() + pageCount, _ := getStarredPageCount(c.Username) + + // diparallel aja biar cepet. + for page := 0; page <= pageCount; page++ { + rc.wg.Add(1) + go func(idx int, stack *requestControl) { + defer stack.wg.Done() + + res, err := c.ListStarred(idx) + if err != nil { + stack.mu.Lock() + stack.err = append(stack.err, err) + stack.mu.Unlock() + return + } + + // ga boleh rebutan, nanti dimarahin mama. + stack.mu.Lock() + stack.starred = append(stack.starred, res...) + stack.mu.Unlock() + }(page, rc) + } + + // tungguin sampe goroutine kelar semua. + rc.wg.Wait() + + if len(rc.err) > 0 { + fmt.Printf("got %d error", len(rc.err)) + fmt.Println(rc.err) + } + + db, err := OpenDB() + if err != nil { + err = fmt.Errorf("failed opening DB: %s", err.Error()) + return err + } + + defer db.Close() + + err = db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte("kleng")) + if err != nil { + return err + } + + j, err := json.Marshal(rc.starred) + if err != nil { + return fmt.Errorf("cannot marshal data: %s", err.Error()) + } + + err = bucket.Put([]byte("starred_repo"), j) + return err + }) + + err = db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte("kleng")) + if bucket == nil { + return fmt.Errorf("Bucket %q not found!", []byte("kleng")) + } + + val := bucket.Get([]byte("starred_repo")) + fmt.Println(string(val)) + + return nil + }) + + // j, err := json.Marshal(rc.starred) + // err = ioutil.WriteFile("./new-starred-1.json", j, 0644) + // if err != nil { + // return err + // } + + return nil +} + +func populationUpdate() error { + return nil +} + +func getStarredPageCount(username string) (count int, err error) { + client := http.DefaultClient + resp, err := client.Get( + fmt.Sprintf("%s/%s/%s", "https://api.github.com/users", username, "starred"), + ) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + links := resp.Header.Get("Link") + if len(links) > 0 { + lastpage := strings.Replace( + strings.Replace( + strings.Split(links, ";")[1], + "rel=\"next\", <", + "", + -1, + ), + ">", + "", + -1, + ) + + lasturl, err := url.Parse(lastpage) + if err != nil { + return count, err + } + + count, err = strconv.Atoi(lasturl.Query().Get("page")) + } + + return count, err +} From 199f69db41b860c33ddf9a0c7765c3475c441ffb Mon Sep 17 00:00:00 2001 From: widnyana Date: Mon, 20 Mar 2017 19:35:35 +0700 Subject: [PATCH 3/5] remove unused code --- utils/gh.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/utils/gh.go b/utils/gh.go index 1b2ee50..be4b2fa 100644 --- a/utils/gh.go +++ b/utils/gh.go @@ -46,11 +46,3 @@ func (g GithubClient) ListStarred(page int) ([]*github.StarredRepository, error) return stared, err } - -func (g GithubClient) PopulateStarred() error { - // for { - /// todo - // } - - return nil -} From 10b177fba29bd5f33347abc2e3f3cdb90ba4829b Mon Sep 17 00:00:00 2001 From: widnyana Date: Mon, 20 Mar 2017 19:36:37 +0700 Subject: [PATCH 4/5] populate trigged by flag --- main.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index c42bcae..17bdd29 100644 --- a/main.go +++ b/main.go @@ -1,19 +1,40 @@ package main import ( + "flag" "fmt" + "log" "os" + "github.com/redite/kleng/core" "github.com/redite/kleng/server" "github.com/redite/kleng/utils" ) +var ( + mode = flag.String("M", "", "run with `mode`. available mode: 'populate', 'http'") +) + func main() { + + flag.Parse() + conf, err := utils.LoadConfig() if err != nil { fmt.Printf("Got error: %s", err.Error()) os.Exit(-1) } - server.RunServer(conf) + switch *mode { + case "http": + log.Fatal(server.RunServer(conf)) + break + case "populate": + err = core.Populate() + if err != nil { + log.Printf("Error Populating: %s\n", err.Error()) + } + break + } + } From 82099eb016fd5c512f15e3342295f40a19f3eca2 Mon Sep 17 00:00:00 2001 From: widnyana Date: Sat, 27 May 2017 22:36:02 +0700 Subject: [PATCH 5/5] add: another mode on Makefile, and default handler on main.go swicth case --- Makefile | 8 +++++--- main.go | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0807d27..cf843d4 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,21 @@ +deps: + @echo "Installing dependencies" + @glide install + build: @echo "Building Kleng" @go build -x -v -race -o kleng run: @echo "Running kleng with HTTP mode" - @make build ./kleng -M http populate: @echo "Running kleng with Populate mode" - @make build ./kleng -M populate clean: @rm -f kleng @go clean all -.PHONY: deps build +.PHONY: deps build populate clean run \ No newline at end of file diff --git a/main.go b/main.go index 17bdd29..f9319a4 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,8 @@ func main() { log.Printf("Error Populating: %s\n", err.Error()) } break + default: + flag.Usage() } }