8 Add support for github markdown emojis

internal/emoji.go

@@ -0,0 +1,130 @@
+package internal
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/gocolly/colly/v2"
+)
+
+const emojiMapTemplate = `// Code generated by cmd/emojiscraper. DO NOT EDIT.
+
+package pkg
+
+var EmojiMap = map[string]string{
+{{- range $key, $value := . }}
+    "{{$key}}": "{{$value}}",
+{{- end }}
+}`
+
+func ScrapeEmojis(emojiOut string, emojiMapOut string) {
+	c := colly.NewCollector(
+		colly.AllowedDomains("gist.github.com"),
+	)
+
+	emojiMap := map[string]string{}
+
+	c.OnHTML("th", func(e *colly.HTMLElement) {
+		code, icon := getEmojiFromHtml(emojiOut, e)
+		if code != "" {
+			emojiMap[code] = icon
+		}
+	})
+
+	c.OnHTML("td", func(e *colly.HTMLElement) {
+		code, icon := getEmojiFromHtml(emojiOut, e)
+		if code != "" {
+			emojiMap[code] = icon
+		}
+	})
+
+	c.OnRequest(func(r *colly.Request) {
+		fmt.Println("Visiting", r.URL.String())
+	})
+
+	if err := c.Visit("https://gist.github.com/rxaviers/7360908"); err != nil {
+		panic(err)
+	}
+
+	if err := createEmojiMapFile(emojiMap, emojiMapOut); err != nil {
+		panic(err)
+	}
+}
+
+func getEmojiFromHtml(outPath string, e *colly.HTMLElement) (string, string) {
+	content := strings.Split(e.Text, " ")
+
+	if e.ChildText("code") == "" {
+		return "", ""
+	}
+
+	emoji := strings.TrimSpace(content[0])
+	if emoji == "" {
+		url := e.ChildAttr("img", "src")
+		path, err := downloadIcon(outPath, url)
+		if err != nil {
+			panic(err)
+		}
+		emoji = path
+	}
+
+	if emoji == content[1] {
+		return "", ""
+	}
+
+	return content[1], emoji
+}
+
+func downloadIcon(baseDir, url string) (string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", fmt.Errorf("failed to download file: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("bad status: %s", resp.Status)
+	}
+
+	parts := strings.Split(url, "/")
+	fileName := parts[len(parts)-1]
+	destPath := filepath.Join(baseDir, fileName)
+	htmlPath := filepath.Join("/static/emojis", fileName)
+
+	if err := os.MkdirAll(filepath.Dir(destPath), os.ModePerm); err != nil {
+		return "", fmt.Errorf("failed to create directories: %v", err)
+	}
+
+	out, err := os.Create(destPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to create file: %v", err)
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to write file: %v", err)
+	}
+
+	return htmlPath, nil
+}
+
+func createEmojiMapFile(emojiMap map[string]string, outputPath string) error {
+	file, err := os.Create(outputPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	tmpl, err := template.New("emojiMap").Parse(emojiMapTemplate)
+	if err != nil {
+		return err
+	}
+
+	return tmpl.Execute(file, emojiMap)
+}