90 lines
3.1 KiB
Go
90 lines
3.1 KiB
Go
package tools
|
|
|
|
import (
|
|
"html"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// htmlPage es una página convertida a texto legible para el modelo.
|
|
type htmlPage struct {
|
|
Title string
|
|
Text string
|
|
Links []string // URLs absolutas, en orden de aparición; [n] en el texto
|
|
}
|
|
|
|
var (
|
|
dropBlocksRe = regexp.MustCompile(`(?is)<(script|style|noscript|svg|head|template|iframe)\b.*?</\s*(script|style|noscript|svg|head|template|iframe)\s*>`)
|
|
commentRe = regexp.MustCompile(`(?s)<!--.*?-->`)
|
|
titleRe = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
|
|
anchorRe = regexp.MustCompile(`(?is)<a\b[^>]*?\bhref\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+))[^>]*>(.*?)</a\s*>`)
|
|
blockTagRe = regexp.MustCompile(`(?i)</?(p|div|br|tr|table|section|article|header|footer|nav|main|aside|h[1-6]|ul|ol|dl|dt|dd|pre|blockquote|hr|form)\b[^>]*>`)
|
|
liTagRe = regexp.MustCompile(`(?i)<li\b[^>]*>`)
|
|
cellOpenRe = regexp.MustCompile(`(?i)<(td|th)\b[^>]*>`)
|
|
cellCloseRe = regexp.MustCompile(`(?i)</(td|th)\s*>`)
|
|
anyTagRe = regexp.MustCompile(`(?s)<[^>]+>`)
|
|
spacesRe = regexp.MustCompile(`[ \t\f\v\x{00a0}]+`)
|
|
blankLinesRe = regexp.MustCompile(`\n[ \t]*(\n[ \t]*)+`)
|
|
)
|
|
|
|
// htmlToText convierte HTML en texto plano conservando la estructura básica
|
|
// (párrafos, listas, tablas) y numerando los enlaces para poder seguirlos.
|
|
//
|
|
// No pretende ser un navegador: su objetivo es que documentación, avisos de
|
|
// seguridad y páginas de issues sean legibles para el modelo con el mínimo
|
|
// de tokens. Se implementa con la biblioteca estándar para no añadir
|
|
// dependencias.
|
|
func htmlToText(src, base string) htmlPage {
|
|
var p htmlPage
|
|
if m := titleRe.FindStringSubmatch(src); m != nil {
|
|
p.Title = strings.TrimSpace(html.UnescapeString(anyTagRe.ReplaceAllString(m[1], "")))
|
|
}
|
|
s := commentRe.ReplaceAllString(src, "")
|
|
s = dropBlocksRe.ReplaceAllString(s, " ")
|
|
|
|
baseURL, _ := url.Parse(base)
|
|
seen := map[string]int{}
|
|
s = anchorRe.ReplaceAllStringFunc(s, func(a string) string {
|
|
m := anchorRe.FindStringSubmatch(a)
|
|
href := html.UnescapeString(m[2] + m[3] + m[4])
|
|
label := strings.TrimSpace(anyTagRe.ReplaceAllString(m[5], ""))
|
|
if strings.HasPrefix(href, "#") || strings.HasPrefix(strings.ToLower(href), "javascript:") || strings.HasPrefix(strings.ToLower(href), "mailto:") {
|
|
return label
|
|
}
|
|
if baseURL != nil {
|
|
if u, err := baseURL.Parse(href); err == nil {
|
|
href = u.String()
|
|
}
|
|
}
|
|
n, ok := seen[href]
|
|
if !ok {
|
|
p.Links = append(p.Links, href)
|
|
n = len(p.Links)
|
|
seen[href] = n
|
|
}
|
|
if label == "" {
|
|
return ""
|
|
}
|
|
return label + " [" + strconv.Itoa(n) + "]"
|
|
})
|
|
|
|
s = liTagRe.ReplaceAllString(s, "\n- ")
|
|
s = cellOpenRe.ReplaceAllString(s, "")
|
|
s = cellCloseRe.ReplaceAllString(s, " | ")
|
|
s = blockTagRe.ReplaceAllString(s, "\n")
|
|
s = anyTagRe.ReplaceAllString(s, "")
|
|
s = html.UnescapeString(s)
|
|
s = spacesRe.ReplaceAllString(s, " ")
|
|
lines := strings.Split(s, "\n")
|
|
for i, l := range lines {
|
|
lines[i] = strings.TrimSuffix(strings.TrimSpace(l), " |")
|
|
lines[i] = strings.TrimSpace(strings.TrimSuffix(lines[i], "|"))
|
|
}
|
|
s = strings.Join(lines, "\n")
|
|
s = blankLinesRe.ReplaceAllString(s, "\n\n")
|
|
p.Text = strings.TrimSpace(s)
|
|
return p
|
|
}
|