Add all flags

Author: Uğur Özyılmazel

pkg/app/app.go

@@ -1,6 +1,8 @@
 package app
 
 import (
+	"bytes"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
@@ -19,29 +21,56 @@ var (
 	optURL                *string
 	optAPIToken           *string
 	optCountryCode        *string
+	optAuthUsername       *string
+	optAuthPassword       *string
+	optCookie             *string
+	optReferer            *string
+	optSelector           *string
 
 	usage = `
-usage: scraper-cli [options...]
+usage: scraper-cli [flags...]
 
   scraper-cli is a command-line interface for Prompt API's Scraper API. Details
   can be found:
 
-  https://promptapi.com/marketplace/description/scraper-api
+      https://promptapi.com/marketplace/description/scraper-api
 
   you need to signup for Prompt API to get your PROMPTAPI_TOKEN. you can signup
   from:
 
-  https://promptapi.com/#signup-form
+      https://promptapi.com/#signup-form
 
   application looks for PROMPTAPI_TOKEN environment variable. if you pass 
   "token" flag, this will override environment variable lookup.
 
-  example token usage:
+  required flag(s):
 
-      $ PROMPTAPI_TOKEN="your-api-key" scraper-cli -url "https://promptapi.com" # or
-      $ scraper-cli -url "https://promptapi.com" -token "your-api-key"
+  -url          web url/address to scrape
+
+
+  optional flags:
+
+  -country      2 character country code
+  -token        promptapi apikey instead of PROMPTAPI_TOKEN env-var
+  -username     for HTTP Realm auth username
+  -password     for HTTP Realm auth password
+  -cookie       URL Encoded cookie header
+  -referer      HTTP referer header
+  -selector     CSS style selector path such as: a.btn div li
+  -version      display version information
+  -help, -h     display help
+
+
+  examples:
+
+  $ scraper-cli -help
+  $ scraper-cli -url "https://promptapi.com"
+  $ scraper-cli -url "https://promptapi.com" -country "EE"
+  $ scraper-cli -url "https://promptapi.com" -country "EE" -selector "a.btn div li"
+
+  $ PROMPTAPI_TOKEN="your-api-key" scraper-cli -url "https://promptapi.com"
+  $ scraper-cli -url "https://promptapi.com" -token "your-api-key"
 
-  options:
 
 `
 )
@@ -55,12 +84,16 @@ type CLIApplication struct {
 func NewCLIApplication() *CLIApplication {
 	flag.Usage = func() {
 		fmt.Fprint(os.Stderr, usage)
-		flag.PrintDefaults()
 	}
 	optVersionInformation = flag.Bool("version", false, "display version information")
 	optURL = flag.String("url", "", "web url/address to scrape")
 	optAPIToken = flag.String("token", "n/a", "use this flag to override PROMPTAPI_TOKEN environment variable")
-	optCountryCode = flag.String("country", "n/a", "2 character country code.")
+	optCountryCode = flag.String("country", "n/a", "2 character country code")
+	optAuthUsername = flag.String("username", "n/a", "for HTTP Realm auth username")
+	optAuthPassword = flag.String("password", "n/a", "for HTTP Realm auth password")
+	optCookie = flag.String("cookie", "n/a", "URL Encoded cookie header")
+	optReferer = flag.String("referer", "n/a", "HTTP referer header")
+	optSelector = flag.String("selector", "n/a", "CSS style selector path such as: a.btn div li")
 
 	flag.Parse()
 
@@ -96,6 +129,10 @@ func (c *CLIApplication) Validate() error {
 		return err
 	}
 
+	if *optAPIToken != "n/a" {
+		os.Setenv("PROMPTAPI_TOKEN", *optAPIToken)
+	}
+
 	return nil
 }
 
@@ -109,14 +146,42 @@ func (c *CLIApplication) Scrape() error {
 
 	if *optCountryCode != "n/a" {
 		params.Country = *optCountryCode
-		fmt.Printf("%v\n", params.Country)
+	}
+	if *optAuthUsername != "n/a" {
+		params.AuthUsername = *optAuthUsername
+	}
+	if *optAuthPassword != "n/a" {
+		params.AuthPassword = *optAuthPassword
+	}
+	if *optCookie != "n/a" {
+		params.Cookie = *optCookie
+	}
+	if *optReferer != "n/a" {
+		params.Referer = *optReferer
+	}
+	if *optSelector != "n/a" {
+		params.Selector = *optSelector
 	}
 
 	result := new(scraper.Result)
 	err := s.Scrape(params, result)
 	if err != nil {
 		return err
 	}
-	fmt.Fprintf(c.Out, "Content-Length: %v", result.Headers["Content-Length"])
+
+	outData := result.Data
+
+	if len(result.DataSelector) > 0 {
+		buffer := new(bytes.Buffer)
+		encoder := json.NewEncoder(buffer)
+		encoder.SetEscapeHTML(false)
+		err := encoder.Encode(result.DataSelector)
+		if err != nil {
+			return err
+		}
+		outData = buffer.String()
+	}
+
+	fmt.Fprintf(c.Out, outData)
 	return nil
 }