功能

• 根據靜態(tài)服務器上基礎的url，獲取所有的文件

運用到的知識點

• 日志處理
  • 能在控制臺打印，又能寫入文件
• 向服務端發(fā)送get請求
  • http.Get(url)
• 遞歸獲取文件夾和創(chuàng)建文件夾

package mainimport (? "fmt"? "io"? "io/ioutil"? "log"? "net/http"? "os"? "regexp"? "strings"? "time")var (? baseurl = "基礎url" ? basePath = "D:\\staticFile" //本地地址? logger *log.Logger)func init(){? fmt.Println("創(chuàng)建日記錄日志文件")? f,err:=os.OpenFile("Log.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 666)? if err!=nil{? log.Fatal("os.OpenFile err",err)? }? writers := []io.Writer{? f,? os.Stdout}? //defer f.Close() 因為這里，就已經將文件關閉了，看來不能夠隨便使用defer? fileAndStdoutWriter := io.MultiWriter(writers...)? logger = log.New(fileAndStdoutWriter, "", log.Ldate|log.Ltime|log.Lshortfile)? logger.Println("---> logger:check to make sure is works")}func main(){? logger.Println("開始計算時間")? t1:=time.Now()? HandleFile(baseurl)? t2:=time.Now()? logger.Println("下載所有的文件總耗時：",t2.Sub(t1))}//遍歷指定路徑下的所有文件，filepath.Walk("路徑",walkFunc)func walkFunc(path string,info os.FileInfo,err error)error{? fmt.Println(path)? return nil}//處理文件信息func HandleFile(url string)error{? if url==""{? err:=fmt.Errorf("url是空的")? logger.Println(err)? return err? }? logger.Println("發(fā)起get請求")? htmlData,err:= HttpGet(url)? if htmlData==""{? err:=fmt.Errorf("HttpGet中有錯啦，請檢查HttpGet數據")? logger.Println(err)? return err? }? if err!=nil{? logger.Println(err)? return err? }? cutUrl := strings.Split(url,"基礎url")[1]? logger.Println("正則匹配信息")? re:=regexp.MustCompile(<a href="(.*?)">(.*?)</a>)? result := re.FindAllStringSubmatch(htmlData,-1)? if result==nil{? err:=fmt.Errorf("正則匹配的數據為空")? logger.Println(err)? return err? }? logger.Println("正則匹配數據")? for i:=0;i<len(result);i++{? publicPath := cutUrl? if len(result[i])!=3{? err:=fmt.Errorf("正則出來的東西不是我想要的")? logger.Println(err)? return err? }? fmt.Println(result[i][2])? if strings.Contains(result[i][2],"/"){? logger.Println(result[i][2]+"是個文件夾")? floderName:=strings.Replace(result[i][2],"/","",-1)? if floderName!=result[i][2]{? publicPath = publicPath+"/"+floderName? fmt.Println("publicPath=============>",publicPath)? logger.Println("創(chuàng)建文件夾")? err=CreateFloder(publicPath)? if err!=nil{? logger.Println("CreateFloder error",err)? return err? }? urlPath := baseurl+publicPath+"/"? fmt.Println("urlPath============>",urlPath)? logger.Println("遞歸")? HandleFile(urlPath)? }? }else{? publicPath = publicPath+result[i][2]? logger.Println("遇到是文件，就將數據寫入文件")? err:=WriteFile(publicPath)? if err!=nil{? logger.Println(err)? return err? }? }? }? return nil}//發(fā)起get請求func HttpGet(url string)(string,error){? if url==""{? err:= fmt.Errorf("傳入的url為空")? logger.Println(err)? return "",err? }? if !strings.Contains(url,"http:"){? err:=fmt.Errorf("傳入的url不正確")? logger.Println(err)? return "",err? }? resp,err:=http.Get(url)? if err!=nil{? err1:= fmt.Errorf("http.Get error===========>%v",err)? logger.Println(err1)? return "",err1? }? defer resp.Body.Close()? body,err := ioutil.ReadAll(resp.Body)? if err!=nil{? err1:=fmt.Errorf("ioutil.ReadAll error===========>%v",err)? logger.Println(err1)? return "",err1? }? if strings.Contains(string(body),"404 page not found"){? err:=fmt.Errorf("找不到該網頁")? logger.Println(err)? return "",err? }? return string(body),nil}//判斷文件是否存在func IsExist(path string)(bool){? if path==""{? return false? }? _,err:=os.Stat(path)? if err!=nil{? if os.IsExist(err){? return true? }else{? return false? }? }? return true}//創(chuàng)建文件夾func CreateFloder(publicPath string)(error){? if publicPath==""{? err:=fmt.Errorf("傳入urlpath的地址空")? logger.Println(err)? return err? }? logger.Println(publicPath,"創(chuàng)建文件夾")? err:=os.MkdirAll(basePath+"//"+publicPath,os.ModePerm)? if err!=nil{? err:=fmt.Errorf("創(chuàng)建文件錯誤啦:%v",err)? logger.Println(err)? return err? }? return nil}//寫入文件func WriteFile(publicPath string)error{? if publicPath==""{? err:=fmt.Errorf("傳入的參數為空，請注意！！！")? logger.Println(err)? return err? }? htmlData,err:=HttpGet(baseurl+publicPath)? if htmlData==""{? err:=fmt.Errorf("HttpGet中有錯啦，請檢查HttpGet數據")? logger.Println(err)? return err? }? if err!=nil{? logger.Println(err)? return err? }? file,err:=os.Create(basePath+publicPath)? defer file.Close()? if err!=nil{? err:=fmt.Errorf("os.Open失敗,err=%v",err)? logger.Println(err)? return err? }? if file==nil{? err:=fmt.Errorf("創(chuàng)建文件失敗")? logger.Println(err)? return err? }? file.WriteString(htmlData)? return nil}

轉載于:https://www.cnblogs.com/MyUniverse/p/11470434.html

總結

以上是生活随笔為你收集整理的golang 根据基础的url下载静态服务器上所有的文件的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。