goquery/example_test.go

package goquery_test

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// This example scrapes the reviews shown on the home page of metalsucks.net.
func Example() {
	// Request the HTML page.
	res, err := http.Get("http://metalsucks.net")
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
	}

	// Load the HTML document
	doc, err := goquery.NewDocumentFromReader(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	// Find the review items
	doc.Find(".sidebar-reviews article .content-block").Each(func(i int, s *goquery.Selection) {
		// For each item found, get the band and title
		band := s.Find("a").Text()
		title := s.Find("i").Text()
		fmt.Printf("Review %d: %s - %s\n", i, band, title)
	})
	// To see the output of the Example while running the test suite (go test), simply
	// remove the leading "x" before Output on the next line. This will cause the
	// example to fail (all the "real" tests should pass).

	// xOutput: voluntarily fail the Example output.
}

// This example shows how to use NewDocumentFromReader from a file.
func ExampleNewDocumentFromReader_file() {
	// create from a file
	f, err := os.Open("some/file.html")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	doc, err := goquery.NewDocumentFromReader(f)
	if err != nil {
		log.Fatal(err)
	}
	// use the goquery document...
	_ = doc.Find("h1")
}

// This example shows how to use NewDocumentFromReader from a string.
func ExampleNewDocumentFromReader_string() {
	// create from a string
	data := `
<html>
	<head>
		<title>My document</title>
	</head>
	<body>
		<h1>Header</h1>
	</body>
</html>`

	doc, err := goquery.NewDocumentFromReader(strings.NewReader(data))
	if err != nil {
		log.Fatal(err)
	}
	header := doc.Find("h1").Text()
	fmt.Println(header)

	// Output: Header
}

func ExampleSingle() {
	html := `
<html>
  <body>
    <div>1</div>
    <div>2</div>
    <div>3</div>
  </body>
</html>
`
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
	if err != nil {
		log.Fatal(err)
	}

	// By default, the selector string selects all matching nodes
	multiSel := doc.Find("div")
	fmt.Println(multiSel.Text())

	// Using goquery.Single, only the first match is selected
	singleSel := doc.FindMatcher(goquery.Single("div"))
	fmt.Println(singleSel.Text())

	// Output:
	// 123
	// 1
}

// This example shows how to use the goquery.Text function to extract clean,
// human-readable text from a selection, similar to BeautifulSoup's get_text.
func ExampleText() {
	page := `
<html>
  <body>
    <div id="content">
      <h1>  Hello  </h1>
      <p>world</p>
      <script>var ignored = 1;</script>
    </div>
  </body>
</html>
`
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(page))
	if err != nil {
		log.Fatal(err)
	}

	// Trim each text node, join the remaining ones with a space, and skip the
	// text of <script> and <style> elements.
	text := goquery.Text(doc.Find("#content"), &goquery.TextOptions{
		Separator: " ",
		Trim:      true,
		Keep: func(n *html.Node) bool {
			if p := n.Parent; p != nil && p.Type == html.ElementNode {
				return p.Data != "script" && p.Data != "style"
			}
			return true
		},
	})
	fmt.Println(text)

	// Output:
	// Hello world
}