初始化Go学习项目

go-crawler/crawler_basic.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"sync"
+)
+
+func main() {
+	urls := []string{
+		"https://golang.org",
+		"https://go.dev",
+		"https://www.baidu.com",
+		"https://www.bing.com",
+	}
+
+	jobs := make(chan string)
+	results := make(chan string)
+
+	var wg sync.WaitGroup
+	workerCount := 3
+
+	//启动 worker
+	for i := 0; i < workerCount; i++ {
+		wg.Add(1)
+		go worker(i, jobs, results, &wg)
+	}
+
+	//发送任务
+	go func() {
+		for _, url := range urls {
+			jobs <- url
+		}
+		close(jobs) // 关闭 channel
+	}()
+
+	//单独goroutine 负责所有 worker结束后 关闭 results
+	go func() {
+		wg.Wait()
+		close(results)
+	}()
+
+	for res := range results {
+		fmt.Println(res)
+	}
+}
+
+// worker 从 jobs 读取 URL，写结果到 results
+func worker(id int, jobs <-chan string, results chan<- string, wg *sync.WaitGroup) {
+	defer wg.Done()
+	for url := range jobs {
+		title, err := fetchTitle(url)
+		if err != nil {
+			results <- fmt.Sprintf("[worker-%d] %s ERROR: %v", id, url, err)
+			continue
+		}
+		results <- fmt.Sprintf("[worker-%d] %s => %s", id, url, title)
+	}
+}
+
+func fetchTitle(url string) (string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	re := regexp.MustCompile(`<title>(.+)</title>`)
+	matches := re.FindSubmatch(body)
+	if len(matches) >= 2 {
+		return string(matches[1]), nil
+	}
+	return "(no title)", nil
+}