package tools import ( "html" "net/url" "regexp" "strconv" "strings" ) // htmlPage es una página convertida a texto legible para el modelo. type htmlPage struct { Title string Text string Links []string // URLs absolutas, en orden de aparición; [n] en el texto } var ( dropBlocksRe = regexp.MustCompile(`(?is)<(script|style|noscript|svg|head|template|iframe)\b.*?`) commentRe = regexp.MustCompile(`(?s)`) titleRe = regexp.MustCompile(`(?is)]*>(.*?)`) anchorRe = regexp.MustCompile(`(?is)]*?\bhref\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+))[^>]*>(.*?)`) blockTagRe = regexp.MustCompile(`(?i)]*>`) liTagRe = regexp.MustCompile(`(?i)]*>`) cellOpenRe = regexp.MustCompile(`(?i)<(td|th)\b[^>]*>`) cellCloseRe = regexp.MustCompile(`(?i)`) anyTagRe = regexp.MustCompile(`(?s)<[^>]+>`) spacesRe = regexp.MustCompile(`[ \t\f\v\x{00a0}]+`) blankLinesRe = regexp.MustCompile(`\n[ \t]*(\n[ \t]*)+`) ) // htmlToText convierte HTML en texto plano conservando la estructura básica // (párrafos, listas, tablas) y numerando los enlaces para poder seguirlos. // // No pretende ser un navegador: su objetivo es que documentación, avisos de // seguridad y páginas de issues sean legibles para el modelo con el mínimo // de tokens. Se implementa con la biblioteca estándar para no añadir // dependencias. func htmlToText(src, base string) htmlPage { var p htmlPage if m := titleRe.FindStringSubmatch(src); m != nil { p.Title = strings.TrimSpace(html.UnescapeString(anyTagRe.ReplaceAllString(m[1], ""))) } s := commentRe.ReplaceAllString(src, "") s = dropBlocksRe.ReplaceAllString(s, " ") baseURL, _ := url.Parse(base) seen := map[string]int{} s = anchorRe.ReplaceAllStringFunc(s, func(a string) string { m := anchorRe.FindStringSubmatch(a) href := html.UnescapeString(m[2] + m[3] + m[4]) label := strings.TrimSpace(anyTagRe.ReplaceAllString(m[5], "")) if strings.HasPrefix(href, "#") || strings.HasPrefix(strings.ToLower(href), "javascript:") || strings.HasPrefix(strings.ToLower(href), "mailto:") { return label } if baseURL != nil { if u, err := baseURL.Parse(href); err == nil { href = u.String() } } n, ok := seen[href] if !ok { p.Links = append(p.Links, href) n = len(p.Links) seen[href] = n } if label == "" { return "" } return label + " [" + strconv.Itoa(n) + "]" }) s = liTagRe.ReplaceAllString(s, "\n- ") s = cellOpenRe.ReplaceAllString(s, "") s = cellCloseRe.ReplaceAllString(s, " | ") s = blockTagRe.ReplaceAllString(s, "\n") s = anyTagRe.ReplaceAllString(s, "") s = html.UnescapeString(s) s = spacesRe.ReplaceAllString(s, " ") lines := strings.Split(s, "\n") for i, l := range lines { lines[i] = strings.TrimSuffix(strings.TrimSpace(l), " |") lines[i] = strings.TrimSpace(strings.TrimSuffix(lines[i], "|")) } s = strings.Join(lines, "\n") s = blankLinesRe.ReplaceAllString(s, "\n\n") p.Text = strings.TrimSpace(s) return p }