一个并发查询A记录的工具导致cpu大量占用的问题

如题,最近在写一个高并发查询大量域名A记录的工具,使用的是github.com/miekg/dns这个包,基本思路是给定一个域名列表和dns服务器列表,将dns服务器做成循环链表,将dns服务器与域名一一对应存入map然后交给worker解析,当域名大约在10万以上的,goroutine在2000左右就会出现大量占用cpu的情况。请问各位有没有写过类似程序的经验,请教一下应该是哪里出问题了。代码如下。

package main

import (
	"bufio"
	"container/ring"
	"errors"
	"fmt"
	"github.com/miekg/dns"
	cmap "github.com/orcaman/concurrent-map"
	"log"
	"os"
	"time"
)

var (
	client = new(dns.Client)

)
var (
	signalsnum = 0
)

func init()  {
	client.Timeout = time.Duration(500*time.Millisecond)
}
func lookupA(fqdn, serverAddr string) ([]string, error) {

	var ips []string
	m1 := new(dns.Msg)
	m1.Id = dns.Id()
	m1.RecursionDesired = true
	m1.Question = make([]dns.Question, 1)
	m1.Question[0] = dns.Question{dns.Fqdn(fqdn), dns.TypeA, dns.ClassINET}
	in, _, err := client.Exchange(m1, serverAddr+":53")
	if err != nil {
		return ips,err
	}
	if len(in.Answer) < 1 {
		return ips, errors.New("no answer")
	}
	for _, answer := range in.Answer {
		if a, ok := answer.(*dns.A); ok {
			ips = append(ips, a.A.String())
			return ips,nil
		}
	}
	return ips, nil
}

func worker(dnsmapkey chan string,gather chan string,m cmap.ConcurrentMap) {
	for  {
		select {
		case name := <-dnsmapkey:
			dnskey, _ := m.Get(name)
			dnserver := fmt.Sprintf("%s", dnskey)
			//fmt.Println(name,"========",dnserver)
			result, err := lookupA(name+".lenovo.com", dnserver)
			if err != nil {
				//有err就认为解析失败,加入失败队列。
			}
			if len(result) > 0 {
				signalsnum++
				gather <- name + "===" + result[0] + "===" + dnserver
			}
			m.Remove(name)
		}


	}
}
func filelines(name string) int {
	line := 0
	file,err := os.Open(name)
	if err != nil{
		log.Fatal(err)
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	for scanner.Scan(){
		line++
	}
	return line
}

var (
	hashMapSize = 3000
)
func main() {
	//profile.Start().Stop()
	r := ring.New(filelines("../dns.txt"))
	//var results []string
	dnsmapkey := make(chan string, hashMapSize)
	m := cmap.New()
	gather := make(chan string)
	fh, err := os.Open("./hostname.txt")
	if err != nil {
		panic(err)
	}
	defer fh.Close()
	dnsfile, err := os.Open("../dns.txt")
	if err != nil {
		panic(err)
	}
	defer dnsfile.Close()
	namescanner := bufio.NewScanner(fh)
	dnscanner := bufio.NewScanner(dnsfile)
	for dnscanner.Scan(){
		r.Value = dnscanner.Text()
		r = r.Next()
	}

	for i := 0; i < hashMapSize; i++ {
		go worker(dnsmapkey,gather,m)
	}

	for namescanner.Scan() {
		name := namescanner.Text()
		dns := fmt.Sprintf("%s",r.Value)
		m.Set(name,dns)
		dnsmapkey <- name
		r = r.Next()
	}
	close(dnsmapkey)
	for i:=0;i<signalsnum;i++{
		fmt.Println(<-gather)
	}


}

共 0 个回复