131 lines
2.6 KiB
Go
131 lines
2.6 KiB
Go
package internal
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
"text/template"
|
|
|
|
"github.com/gocolly/colly/v2"
|
|
)
|
|
|
|
const emojiMapTemplate = `// Code generated by cmd/emojiscraper. DO NOT EDIT.
|
|
|
|
package pkg
|
|
|
|
var EmojiMap = map[string]string{
|
|
{{- range $key, $value := . }}
|
|
"{{$key}}": "{{$value}}",
|
|
{{- end }}
|
|
}`
|
|
|
|
func ScrapeEmojis(emojiOut string, emojiMapOut string) {
|
|
c := colly.NewCollector(
|
|
colly.AllowedDomains("gist.github.com"),
|
|
)
|
|
|
|
emojiMap := map[string]string{}
|
|
|
|
c.OnHTML("th", func(e *colly.HTMLElement) {
|
|
code, icon := getEmojiFromHtml(emojiOut, e)
|
|
if code != "" {
|
|
emojiMap[code] = icon
|
|
}
|
|
})
|
|
|
|
c.OnHTML("td", func(e *colly.HTMLElement) {
|
|
code, icon := getEmojiFromHtml(emojiOut, e)
|
|
if code != "" {
|
|
emojiMap[code] = icon
|
|
}
|
|
})
|
|
|
|
c.OnRequest(func(r *colly.Request) {
|
|
fmt.Println("Visiting", r.URL.String())
|
|
})
|
|
|
|
if err := c.Visit("https://gist.github.com/rxaviers/7360908"); err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
if err := createEmojiMapFile(emojiMap, emojiMapOut); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
func getEmojiFromHtml(outPath string, e *colly.HTMLElement) (string, string) {
|
|
content := strings.Split(e.Text, " ")
|
|
|
|
if e.ChildText("code") == "" {
|
|
return "", ""
|
|
}
|
|
|
|
emoji := strings.TrimSpace(content[0])
|
|
if emoji == "" {
|
|
url := e.ChildAttr("img", "src")
|
|
path, err := downloadIcon(outPath, url)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
emoji = path
|
|
}
|
|
|
|
if emoji == content[1] {
|
|
return "", ""
|
|
}
|
|
|
|
return content[1], emoji
|
|
}
|
|
|
|
func downloadIcon(baseDir, url string) (string, error) {
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to download file: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return "", fmt.Errorf("bad status: %s", resp.Status)
|
|
}
|
|
|
|
parts := strings.Split(url, "/")
|
|
fileName := parts[len(parts)-1]
|
|
destPath := path.Join(baseDir, fileName)
|
|
htmlPath := path.Join("/static/emojis", fileName)
|
|
|
|
if err := os.MkdirAll(path.Dir(destPath), os.ModePerm); err != nil {
|
|
return "", fmt.Errorf("failed to create directories: %v", err)
|
|
}
|
|
|
|
out, err := os.Create(destPath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to create file: %v", err)
|
|
}
|
|
defer out.Close()
|
|
|
|
_, err = io.Copy(out, resp.Body)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to write file: %v", err)
|
|
}
|
|
|
|
return htmlPath, nil
|
|
}
|
|
|
|
func createEmojiMapFile(emojiMap map[string]string, outputPath string) error {
|
|
file, err := os.Create(outputPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
|
|
tmpl, err := template.New("emojiMap").Parse(emojiMapTemplate)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return tmpl.Execute(file, emojiMap)
|
|
}
|