migrate to normal Github api and fix projects count bug

platform/gitee.go

@@ -62,6 +62,7 @@ func GiteeCommand() *gcli.Command {
 
 	// bind args with names
 	gitee.AddArg("repoSource", "Tell me which repo dir or list your want to sync, is required", false)
+	gitee.AddArg("token", "Provide platform token for skip api rate limit", false)
 
 	return gitee
 }
@@ -74,7 +75,7 @@ func syncGitee(c *gcli.Command, args []string) error {
 	}
 
 	// check repodir and print projects to ensure
-	repos, source, orgPath := share.ReadyToAuth(args[0], c.Name)
+	repos, source, orgPath := share.ReadyToAuth(args, c.Name)
 	if repos == nil {
 		return nil
 	}

share/tools.go

@@ -113,7 +113,7 @@ func isGitRepo(repoPath string) (isGit bool) {
 }
 
 func printRepos(repos []RepoLocal) {
-	color.Yellow.Println(len(repos), "repositories detected, please check below: ", "\n")
+	color.Yellow.Printf("\n%d repositories detected, please check below:\n\n", len(repos))
 
 	alertFlag := false
 
@@ -153,24 +153,29 @@ func getRepoLocal(repos []string) (reposLocal []RepoLocal) {
 	return reposLocal
 }
 
-func getRepoApi(repoDir string) (reposLocal []RepoLocal, repos []string) {
+func getRepoApi(repoDir, tokenFix string) (reposLocal []RepoLocal, repos []string) {
+	var orgType string
+	var repoArray []map[string]interface{}
 	orgPath := repoDir[7:]
 	platform := repoDir[:6]
-	apiUrl := fmt.Sprintf("https://api.github.com/search/repositories?q=org:%s", orgPath)
-	result, err := GetByte(apiUrl)
-	if err != nil {
-		color.Red.Printf("Request error: %s \n", err.Error())
+
+	// check path type
+	orgType = getGithubType(orgPath, tokenFix)
+	if orgType == "" {
 		return nil, nil
 	}
-	jsonRes, _ := simplejson.NewJson([]byte(result))
-	count, _ := jsonRes.Get("total_count").Float64()
+
+	color.Green.Printf("Selected %s, type is %s, Start to fetch repository list...\n", orgPath, orgType)
+
+	// get repos
+	getGithubRepo(orgPath, orgType, &repoArray, "", tokenFix)
+
+	count := len(repoArray)
 	if count == 0 {
-		color.Yellow.Printf("No repositories found in https://github.com/%s\nPlease check if path is validate\n", orgPath)
+		color.Yellow.Printf("No repositories found in https://github.com/%s\nPlease check if path is valid\n", orgPath)
 		return nil, nil
 	} else { // TODO optimize code with local repo and struct repos
-		repoArray, _ := jsonRes.Get("items").Array()
-		for _, ra := range repoArray {
-			item := ra.(map[string]interface{})
+		for _, item := range repoArray {
 			outOf1G := false
 			size, _ := item["size"].(json.Number).Int64()
 			if size > 1024*1024 {
@@ -195,6 +200,75 @@ func getRepoItemWorker(paths <- chan string, wp *sync.WaitGroup, reposLocal *[]R
 	}
 }
 
+func errGithub(body  *simplejson.Json) bool {
+	mesg, err := body.Get("message").String()
+	if err != nil {
+		return false
+	} else {
+		color.Red.Println(mesg)
+		return true
+	}
+}
+
+func getGithubType(orgPath, tokenFix string) string {
+	typeUrl := fmt.Sprintf("https://api.github.com/users/%s?from=up2Gitx%s", orgPath, tokenFix)
+	result, _, err := GetByte(typeUrl)
+	if err != nil {
+		color.Red.Printf("Request Type error: %s \n", err.Error())
+		return ""
+	}
+
+	typeRes, _ := simplejson.NewJson([]byte(result))
+	errGithub(typeRes)
+
+	typeStr, _ := typeRes.Get("type").String()
+	if typeStr == "User" {
+		return "users"
+	} else if typeStr == "Organization" {
+		 return "orgs"
+	} else {
+		color.Red.Printf("%s not exists, please enter a valid path!\n", orgPath)
+		return ""
+	}
+}
+
+func getGithubRepo(orgPath string, orgType string, repoArray *[]map[string]interface{}, url string, tokenFix string) {
+	var apiUrl string
+	if url != "" {
+		apiUrl = url
+	} else {
+		apiUrl = fmt.Sprintf("https://api.github.com/%s/%s/repos?per_page=20%s", orgType, orgPath, tokenFix)
+	}
+	fmt.Printf("Fetching %s \n", apiUrl)
+
+	result, header, err := GetByte(apiUrl)
+	if err != nil {
+		color.Red.Printf("Request Repositories error: %s \n", err.Error())
+	}
+
+	jsonRes, _ := simplejson.NewJson([]byte(result))
+	errGithub(jsonRes)
+
+	// TODO optimize type assertion remove for
+	tmpRepos, _ := jsonRes.Array()
+	for _, re := range tmpRepos {
+		*repoArray = append(*repoArray, re.(map[string]interface {}))
+	}
+
+	pageLink := header.Get("link")
+	if pageLink != "" {
+		link := strings.Split(pageLink, ",")
+		for _, l := range link {
+			kv := strings.Split(l, ";")
+			if strings.Contains(kv[1], "next") {
+				getGithubRepo(orgPath, orgType, repoArray, kv[0][1 : len(kv[0])-1], "")
+				break
+			}
+		}
+	}
+
+}
+
 func repoSize(path string) (float32, bool, error) {
 	var size int64
 	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
@@ -212,11 +286,18 @@ func repoSize(path string) (float32, bool, error) {
 	return sizeMB, outOf1G, err
 }
 
-func ReadyToAuth(repoDir string, platform string) ([]string, string, string) {
+func ReadyToAuth(args []string, platform string) ([]string, string, string) {
+	repoDir := args[0]
+
+	// if up to gitee from github|gitlab|bitbucket...
 	if platform == "gitee" {
 		reg, err := regexp.Compile(`^github:.+`)
 		if err == nil && reg.MatchString(repoDir) {
-			reposLocal, repos := getRepoApi(repoDir)
+			var tokenFix string
+			if len(args) == 2 && args[1] != "" {
+				tokenFix = fmt.Sprintf("&access_token=%s", args[1])
+			}
+			reposLocal, repos := getRepoApi(repoDir, tokenFix)
 			if repos != nil {
 				printRepos(reposLocal)
 				if len(repos) > 1000 {
@@ -257,16 +338,17 @@ func ExitMessage() {
 	color.Yellow.Println("Bye, see you next time!")
 }
 
-func GetByte(url string) ([]byte, error) {
+func GetByte(url string) ([]byte, NetHttp.Header, error) {
 	response, err := NetHttp.Get(url)
 	if err != nil {
 		color.Red.Printf("Request failed, Error: %s \n", err.Error())
-		return nil, err
+		return nil, nil, err
 	}
 	defer response.Body.Close()
 
 	body, _ := ioutil.ReadAll(response.Body)
-	return body, nil
+	header := response.Header
+	return body, header, nil
 }
 
 func Get(url string) (map[string]interface{}, error) {