Init

2018-03-07 15:31:02 +03:00
parent 8f91676943
commit 26296c2cc7
5 changed files with 296 additions and 0 deletions
@@ -0,0 +1,193 @@
+package utils
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"time"
+)
+
+var agents = []string{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9",
+	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36",
+	"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1"}
+
+func randInt(min int, max int) int {
+	rand.Seed(time.Now().UTC().UnixNano())
+	return min + rand.Intn(max-min)
+}
+
+func randomUserAgent() string {
+	return agents[randInt(0, len(agents))]
+}
+
+func HttpGet(url string, client *http.Client) string {
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("User-Agent", randomUserAgent())
+	req.Header.Set("Accept-Language", "en-us")
+	// req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	// req.Header.Set("Referer", "https://cart.tinydeal.com/index.php?__page_id=checkout&order_source=shopping_cart")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Printf("Cannot get url %s, err: %s", url, err.Error())
+		return ""
+	}
+
+	defer resp.Body.Close()
+	contents, _ := ioutil.ReadAll(resp.Body)
+	return string(contents)
+}
+
+func HttpGetDump(url string, client *http.Client, file string) (string, string) {
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("User-Agent", randomUserAgent())
+	req.Header.Set("Accept-Language", "en-us")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Printf("Cannot get url %s, err: %s", url, err.Error())
+		return "", ""
+	}
+
+	defer resp.Body.Close()
+
+	dump, err := httputil.DumpResponse(resp, true)
+	if err != nil {
+		log.Fatal(err)
+	}
+	dumpStr := string(dump)
+	File_put_contents(file, dumpStr)
+
+	contents, _ := ioutil.ReadAll(resp.Body)
+	return string(contents), dumpStr
+}
+
+func HttpPostJSON(url string, data string, referer_optional string, client *http.Client, isAjax bool) string {
+	req, _ := http.NewRequest("POST", url, bytes.NewBuffer([]byte(data)))
+	req.Header.Set("User-Agent", randomUserAgent())
+	if isAjax {
+		req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	}
+	//	req.Header.Set("Origin", "https://www.topcashback.com")
+	req.Header.Set("Accept-Language", "en-us")
+	req.Header.Set("Content-Type", "application/json")
+	if referer_optional != "" {
+		req.Header.Set("Referer", referer_optional)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Printf("Cannot post url %s, err: %s", url, err.Error())
+		return ""
+	}
+
+	defer resp.Body.Close()
+	contents, _ := ioutil.ReadAll(resp.Body)
+	return string(contents)
+}
+
+func HttpPostString(url string, data string, referer_optional string, client *http.Client, isAjax bool) string {
+	req, _ := http.NewRequest("POST", url, bytes.NewBuffer([]byte(data)))
+	req.Header.Set("User-Agent", randomUserAgent())
+	if isAjax {
+		req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	}
+	req.Header.Set("Accept-Language", "en-us")
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
+	if referer_optional != "" {
+		req.Header.Set("Referer", referer_optional)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Printf("Cannot post url %s, err: %s", url, err.Error())
+		return ""
+	}
+
+	defer resp.Body.Close()
+	contents, _ := ioutil.ReadAll(resp.Body)
+	return string(contents)
+}
+
+func HttpPostStringToken(url string, data string, referer_optional string, token string, client *http.Client, isAjax bool) string {
+	req, _ := http.NewRequest("POST", url, bytes.NewBuffer([]byte(data)))
+	req.Header.Set("User-Agent", randomUserAgent())
+	if isAjax {
+		req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	}
+	req.Header.Set("Accept-Language", "en-us")
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
+	if referer_optional != "" {
+		req.Header.Set("Referer", referer_optional)
+	}
+
+	req.Header.Set("X-CSRF-Token", token)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Printf("Cannot post url %s, err: %s", url, err.Error())
+		return ""
+	}
+
+	defer resp.Body.Close()
+	contents, _ := ioutil.ReadAll(resp.Body)
+	return string(contents)
+}
+
+func HttpPost(url string, data url.Values, client *http.Client) string {
+	req, _ := http.NewRequest("POST", url, bytes.NewBuffer([]byte(data.Encode())))
+	req.Header.Set("User-Agent", randomUserAgent())
+	req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	req.Header.Set("Accept-Language", "en-us")
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
+	req.Header.Set("Referer", "http://user.everbuying.net/m-users-a-sign.htm")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Printf("Cannot post url %s, err: %s", url, err.Error())
+		return ""
+	}
+
+	if resp.StatusCode == 302 && client.CheckRedirect != nil {
+		return resp.Header.Get("Location")
+	}
+
+	defer resp.Body.Close()
+	contents, _ := ioutil.ReadAll(resp.Body)
+	return string(contents)
+}
+
+func HttpDownloadFile(url string, filepath string, client *http.Client) error {
+	// Create the file
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	// Get the data
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("User-Agent", randomUserAgent())
+	req.Header.Set("Accept-Language", "en-us")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Printf("Cannot get url %s, err: %s", url, err.Error())
+		return err
+	}
+	defer resp.Body.Close()
+
+	// Writer the body to file
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+	return nil
+}