1081 lines
40 KiB
Python
1081 lines
40 KiB
Python
"""Beautiful Soup
|
|
Elixir and Tonic
|
|
"The Screen-Scraper's Friend"
|
|
v2.1.1
|
|
http://www.crummy.com/software/BeautifulSoup/
|
|
|
|
Beautiful Soup parses arbitrarily invalid XML- or HTML-like substance
|
|
into a tree representation. It provides methods and Pythonic idioms
|
|
that make it easy to search and modify the tree.
|
|
|
|
A well-formed XML/HTML document will yield a well-formed data
|
|
structure. An ill-formed XML/HTML document will yield a
|
|
correspondingly ill-formed data structure. If your document is only
|
|
locally well-formed, you can use this library to find and process the
|
|
well-formed part of it. The BeautifulSoup class has heuristics for
|
|
obtaining a sensible parse tree in the face of common HTML errors.
|
|
|
|
Beautiful Soup has no external dependencies. It works with Python 2.2
|
|
and up.
|
|
|
|
Beautiful Soup defines classes for four different parsing strategies:
|
|
|
|
* BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific
|
|
language that kind of looks like XML.
|
|
|
|
* BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid
|
|
or invalid.
|
|
|
|
* ICantBelieveItsBeautifulSoup, for parsing valid but bizarre HTML
|
|
that trips up BeautifulSoup.
|
|
|
|
* BeautifulSOAP, for making it easier to parse XML documents that use
|
|
lots of subelements containing a single string, where you'd prefer
|
|
they put that string into an attribute (such as SOAP messages).
|
|
|
|
You can subclass BeautifulStoneSoup or BeautifulSoup to create a
|
|
parsing strategy specific to an XML schema or a particular bizarre
|
|
HTML document. Typically your subclass would just override
|
|
SELF_CLOSING_TAGS and/or NESTABLE_TAGS.
|
|
"""
|
|
from __future__ import generators
|
|
|
|
__author__ = "Leonard Richardson (leonardr@segfault.org)"
|
|
__version__ = "2.1.1"
|
|
__date__ = "$Date: 2004/10/18 00:14:20 $"
|
|
__copyright__ = "Copyright (c) 2004-2005 Leonard Richardson"
|
|
__license__ = "PSF"
|
|
|
|
from sgmllib import SGMLParser, SGMLParseError
|
|
import types
|
|
import re
|
|
import sgmllib
|
|
|
|
#This code makes Beautiful Soup able to parse XML with namespaces
|
|
sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*')
|
|
|
|
class NullType(object):
|
|
|
|
"""Similar to NoneType with a corresponding singleton instance
|
|
'Null' that, unlike None, accepts any message and returns itself.
|
|
|
|
Examples:
|
|
>>> Null("send", "a", "message")("and one more",
|
|
... "and what you get still") is Null
|
|
True
|
|
"""
|
|
|
|
def __new__(cls): return Null
|
|
def __call__(self, *args, **kwargs): return Null
|
|
## def __getstate__(self, *args): return Null
|
|
def __getattr__(self, attr): return Null
|
|
def __getitem__(self, item): return Null
|
|
def __setattr__(self, attr, value): pass
|
|
def __setitem__(self, item, value): pass
|
|
def __len__(self): return 0
|
|
# FIXME: is this a python bug? otherwise ``for x in Null: pass``
|
|
# never terminates...
|
|
def __iter__(self): return iter([])
|
|
def __contains__(self, item): return False
|
|
def __repr__(self): return "Null"
|
|
Null = object.__new__(NullType)
|
|
|
|
class PageElement:
|
|
"""Contains the navigational information for some part of the page
|
|
(either a tag or a piece of text)"""
|
|
|
|
def setup(self, parent=Null, previous=Null):
|
|
"""Sets up the initial relations between this element and
|
|
other elements."""
|
|
self.parent = parent
|
|
self.previous = previous
|
|
self.next = Null
|
|
self.previousSibling = Null
|
|
self.nextSibling = Null
|
|
if self.parent and self.parent.contents:
|
|
self.previousSibling = self.parent.contents[-1]
|
|
self.previousSibling.nextSibling = self
|
|
|
|
def findNext(self, name=None, attrs={}, text=None):
|
|
"""Returns the first item that matches the given criteria and
|
|
appears after this Tag in the document."""
|
|
return self._first(self.fetchNext, name, attrs, text)
|
|
firstNext = findNext
|
|
|
|
def fetchNext(self, name=None, attrs={}, text=None, limit=None):
|
|
"""Returns all items that match the given criteria and appear
|
|
before after Tag in the document."""
|
|
return self._fetch(name, attrs, text, limit, self.nextGenerator)
|
|
|
|
def findNextSibling(self, name=None, attrs={}, text=None):
|
|
"""Returns the closest sibling to this Tag that matches the
|
|
given criteria and appears after this Tag in the document."""
|
|
return self._first(self.fetchNextSiblings, name, attrs, text)
|
|
firstNextSibling = findNextSibling
|
|
|
|
def fetchNextSiblings(self, name=None, attrs={}, text=None, limit=None):
|
|
"""Returns the siblings of this Tag that match the given
|
|
criteria and appear after this Tag in the document."""
|
|
return self._fetch(name, attrs, text, limit, self.nextSiblingGenerator)
|
|
|
|
def findPrevious(self, name=None, attrs={}, text=None):
|
|
"""Returns the first item that matches the given criteria and
|
|
appears before this Tag in the document."""
|
|
return self._first(self.fetchPrevious, name, attrs, text)
|
|
|
|
def fetchPrevious(self, name=None, attrs={}, text=None, limit=None):
|
|
"""Returns all items that match the given criteria and appear
|
|
before this Tag in the document."""
|
|
return self._fetch(name, attrs, text, limit, self.previousGenerator)
|
|
firstPrevious = findPrevious
|
|
|
|
def findPreviousSibling(self, name=None, attrs={}, text=None):
|
|
"""Returns the closest sibling to this Tag that matches the
|
|
given criteria and appears before this Tag in the document."""
|
|
return self._first(self.fetchPreviousSiblings, name, attrs, text)
|
|
firstPreviousSibling = findPreviousSibling
|
|
|
|
def fetchPreviousSiblings(self, name=None, attrs={}, text=None,
|
|
limit=None):
|
|
"""Returns the siblings of this Tag that match the given
|
|
criteria and appear before this Tag in the document."""
|
|
return self._fetch(name, attrs, text, limit,
|
|
self.previousSiblingGenerator)
|
|
|
|
def findParent(self, name=None, attrs={}):
|
|
"""Returns the closest parent of this Tag that matches the given
|
|
criteria."""
|
|
r = Null
|
|
l = self.fetchParents(name, attrs, 1)
|
|
if l:
|
|
r = l[0]
|
|
return r
|
|
firstParent = findParent
|
|
|
|
def fetchParents(self, name=None, attrs={}, limit=None):
|
|
"""Returns the parents of this Tag that match the given
|
|
criteria."""
|
|
return self._fetch(name, attrs, None, limit, self.parentGenerator)
|
|
|
|
#These methods do the real heavy lifting.
|
|
|
|
def _first(self, method, name, attrs, text):
|
|
r = Null
|
|
l = method(name, attrs, text, 1)
|
|
if l:
|
|
r = l[0]
|
|
return r
|
|
|
|
def _fetch(self, name, attrs, text, limit, generator):
|
|
"Iterates over a generator looking for things that match."
|
|
if not hasattr(attrs, 'items'):
|
|
attrs = {'class' : attrs}
|
|
|
|
results = []
|
|
g = generator()
|
|
while True:
|
|
try:
|
|
i = g.next()
|
|
except StopIteration:
|
|
break
|
|
found = None
|
|
if isinstance(i, Tag):
|
|
if not text:
|
|
if not name or self._matches(i, name):
|
|
match = True
|
|
for attr, matchAgainst in attrs.items():
|
|
check = i.get(attr)
|
|
if not self._matches(check, matchAgainst):
|
|
match = False
|
|
break
|
|
if match:
|
|
found = i
|
|
elif text:
|
|
if self._matches(i, text):
|
|
found = i
|
|
if found:
|
|
results.append(found)
|
|
if limit and len(results) >= limit:
|
|
break
|
|
return results
|
|
|
|
#Generators that can be used to navigate starting from both
|
|
#NavigableTexts and Tags.
|
|
def nextGenerator(self):
|
|
i = self
|
|
while i:
|
|
i = i.next
|
|
yield i
|
|
|
|
def nextSiblingGenerator(self):
|
|
i = self
|
|
while i:
|
|
i = i.nextSibling
|
|
yield i
|
|
|
|
def previousGenerator(self):
|
|
i = self
|
|
while i:
|
|
i = i.previous
|
|
yield i
|
|
|
|
def previousSiblingGenerator(self):
|
|
i = self
|
|
while i:
|
|
i = i.previousSibling
|
|
yield i
|
|
|
|
def parentGenerator(self):
|
|
i = self
|
|
while i:
|
|
i = i.parent
|
|
yield i
|
|
|
|
def _matches(self, chunk, howToMatch):
|
|
#print 'looking for %s in %s' % (howToMatch, chunk)
|
|
#
|
|
# If given a list of items, return true if the list contains a
|
|
# text element that matches.
|
|
if isList(chunk) and not isinstance(chunk, Tag):
|
|
for tag in chunk:
|
|
if isinstance(tag, NavigableText) and self._matches(tag, howToMatch):
|
|
return True
|
|
return False
|
|
if callable(howToMatch):
|
|
return howToMatch(chunk)
|
|
if isinstance(chunk, Tag):
|
|
#Custom match methods take the tag as an argument, but all other
|
|
#ways of matching match the tag name as a string
|
|
chunk = chunk.name
|
|
#Now we know that chunk is a string
|
|
if not isinstance(chunk, basestring):
|
|
chunk = str(chunk)
|
|
if hasattr(howToMatch, 'match'):
|
|
# It's a regexp object.
|
|
return howToMatch.search(chunk)
|
|
if isList(howToMatch):
|
|
return chunk in howToMatch
|
|
if hasattr(howToMatch, 'items'):
|
|
return howToMatch.has_key(chunk)
|
|
#It's just a string
|
|
return str(howToMatch) == chunk
|
|
|
|
class NavigableText(PageElement):
|
|
|
|
def __getattr__(self, attr):
|
|
"For backwards compatibility, text.string gives you text"
|
|
if attr == 'string':
|
|
return self
|
|
else:
|
|
raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr)
|
|
|
|
class NavigableString(str, NavigableText):
|
|
pass
|
|
|
|
class NavigableUnicodeString(unicode, NavigableText):
|
|
pass
|
|
|
|
class Tag(PageElement):
|
|
|
|
"""Represents a found HTML tag with its attributes and contents."""
|
|
|
|
def __init__(self, name, attrs=None, parent=Null, previous=Null):
|
|
"Basic constructor."
|
|
self.name = name
|
|
if attrs == None:
|
|
attrs = []
|
|
self.attrs = attrs
|
|
self.contents = []
|
|
self.setup(parent, previous)
|
|
self.hidden = False
|
|
|
|
def get(self, key, default=None):
|
|
"""Returns the value of the 'key' attribute for the tag, or
|
|
the value given for 'default' if it doesn't have that
|
|
attribute."""
|
|
return self._getAttrMap().get(key, default)
|
|
|
|
def __getitem__(self, key):
|
|
"""tag[key] returns the value of the 'key' attribute for the tag,
|
|
and throws an exception if it's not there."""
|
|
return self._getAttrMap()[key]
|
|
|
|
def __iter__(self):
|
|
"Iterating over a tag iterates over its contents."
|
|
return iter(self.contents)
|
|
|
|
def __len__(self):
|
|
"The length of a tag is the length of its list of contents."
|
|
return len(self.contents)
|
|
|
|
def __contains__(self, x):
|
|
return x in self.contents
|
|
|
|
def __nonzero__(self):
|
|
"A tag is non-None even if it has no contents."
|
|
return True
|
|
|
|
def __setitem__(self, key, value):
|
|
"""Setting tag[key] sets the value of the 'key' attribute for the
|
|
tag."""
|
|
self._getAttrMap()
|
|
self.attrMap[key] = value
|
|
found = False
|
|
for i in range(0, len(self.attrs)):
|
|
if self.attrs[i][0] == key:
|
|
self.attrs[i] = (key, value)
|
|
found = True
|
|
if not found:
|
|
self.attrs.append((key, value))
|
|
self._getAttrMap()[key] = value
|
|
|
|
def __delitem__(self, key):
|
|
"Deleting tag[key] deletes all 'key' attributes for the tag."
|
|
for item in self.attrs:
|
|
if item[0] == key:
|
|
self.attrs.remove(item)
|
|
#We don't break because bad HTML can define the same
|
|
#attribute multiple times.
|
|
self._getAttrMap()
|
|
if self.attrMap.has_key(key):
|
|
del self.attrMap[key]
|
|
|
|
def __call__(self, *args, **kwargs):
|
|
"""Calling a tag like a function is the same as calling its
|
|
fetch() method. Eg. tag('a') returns a list of all the A tags
|
|
found within this tag."""
|
|
return apply(self.fetch, args, kwargs)
|
|
|
|
def __getattr__(self, tag):
|
|
if len(tag) > 3 and tag.rfind('Tag') == len(tag)-3:
|
|
return self.first(tag[:-3])
|
|
elif tag.find('__') != 0:
|
|
return self.first(tag)
|
|
|
|
def __eq__(self, other):
|
|
"""Returns true iff this tag has the same name, the same attributes,
|
|
and the same contents (recursively) as the given tag.
|
|
|
|
NOTE: right now this will return false if two tags have the
|
|
same attributes in a different order. Should this be fixed?"""
|
|
if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other):
|
|
return False
|
|
for i in range(0, len(self.contents)):
|
|
if self.contents[i] != other.contents[i]:
|
|
return False
|
|
return True
|
|
|
|
def __ne__(self, other):
|
|
"""Returns true iff this tag is not identical to the other tag,
|
|
as defined in __eq__."""
|
|
return not self == other
|
|
|
|
def __repr__(self):
|
|
"""Renders this tag as a string."""
|
|
return str(self)
|
|
|
|
def __unicode__(self):
|
|
return self.__str__(1)
|
|
|
|
def __str__(self, needUnicode=None, showStructureIndent=None):
|
|
"""Returns a string or Unicode representation of this tag and
|
|
its contents.
|
|
|
|
NOTE: since Python's HTML parser consumes whitespace, this
|
|
method is not certain to reproduce the whitespace present in
|
|
the original string."""
|
|
|
|
attrs = []
|
|
if self.attrs:
|
|
for key, val in self.attrs:
|
|
attrs.append('%s="%s"' % (key, val))
|
|
close = ''
|
|
closeTag = ''
|
|
if self.isSelfClosing():
|
|
close = ' /'
|
|
else:
|
|
closeTag = '</%s>' % self.name
|
|
indentIncrement = None
|
|
if showStructureIndent != None:
|
|
indentIncrement = showStructureIndent
|
|
if not self.hidden:
|
|
indentIncrement += 1
|
|
contents = self.renderContents(indentIncrement, needUnicode=needUnicode)
|
|
if showStructureIndent:
|
|
space = '\n%s' % (' ' * showStructureIndent)
|
|
if self.hidden:
|
|
s = contents
|
|
else:
|
|
s = []
|
|
attributeString = ''
|
|
if attrs:
|
|
attributeString = ' ' + ' '.join(attrs)
|
|
if showStructureIndent:
|
|
s.append(space)
|
|
s.append('<%s%s%s>' % (self.name, attributeString, close))
|
|
s.append(contents)
|
|
if closeTag and showStructureIndent != None:
|
|
s.append(space)
|
|
s.append(closeTag)
|
|
s = ''.join(s)
|
|
isUnicode = type(s) == types.UnicodeType
|
|
if needUnicode and not isUnicode:
|
|
s = unicode(s)
|
|
elif isUnicode and needUnicode==False:
|
|
s = str(s)
|
|
return s
|
|
|
|
def prettify(self, needUnicode=None):
|
|
return self.__str__(needUnicode, showStructureIndent=True)
|
|
|
|
def renderContents(self, showStructureIndent=None, needUnicode=None):
|
|
"""Renders the contents of this tag as a (possibly Unicode)
|
|
string."""
|
|
s=[]
|
|
for c in self:
|
|
text = None
|
|
if isinstance(c, NavigableUnicodeString) or type(c) == types.UnicodeType:
|
|
text = unicode(c)
|
|
elif isinstance(c, Tag):
|
|
s.append(c.__str__(needUnicode, showStructureIndent))
|
|
elif needUnicode:
|
|
text = unicode(c)
|
|
else:
|
|
text = str(c)
|
|
if text:
|
|
if showStructureIndent != None:
|
|
if text[-1] == '\n':
|
|
text = text[:-1]
|
|
s.append(text)
|
|
return ''.join(s)
|
|
|
|
#Soup methods
|
|
|
|
def firstText(self, text, recursive=True):
|
|
"""Convenience method to retrieve the first piece of text matching the
|
|
given criteria. 'text' can be a string, a regular expression object,
|
|
a callable that takes a string and returns whether or not the
|
|
string 'matches', etc."""
|
|
return self.first(recursive=recursive, text=text)
|
|
|
|
def fetchText(self, text, recursive=True, limit=None):
|
|
"""Convenience method to retrieve all pieces of text matching the
|
|
given criteria. 'text' can be a string, a regular expression object,
|
|
a callable that takes a string and returns whether or not the
|
|
string 'matches', etc."""
|
|
return self.fetch(recursive=recursive, text=text, limit=limit)
|
|
|
|
def first(self, name=None, attrs={}, recursive=True, text=None):
|
|
"""Return only the first child of this
|
|
Tag matching the given criteria."""
|
|
r = Null
|
|
l = self.fetch(name, attrs, recursive, text, 1)
|
|
if l:
|
|
r = l[0]
|
|
return r
|
|
findChild = first
|
|
|
|
def fetch(self, name=None, attrs={}, recursive=True, text=None,
|
|
limit=None):
|
|
"""Extracts a list of Tag objects that match the given
|
|
criteria. You can specify the name of the Tag and any
|
|
attributes you want the Tag to have.
|
|
|
|
The value of a key-value pair in the 'attrs' map can be a
|
|
string, a list of strings, a regular expression object, or a
|
|
callable that takes a string and returns whether or not the
|
|
string matches for some custom definition of 'matches'. The
|
|
same is true of the tag name."""
|
|
generator = self.recursiveChildGenerator
|
|
if not recursive:
|
|
generator = self.childGenerator
|
|
return self._fetch(name, attrs, text, limit, generator)
|
|
fetchChildren = fetch
|
|
|
|
#Utility methods
|
|
|
|
def isSelfClosing(self):
|
|
"""Returns true iff this is a self-closing tag as defined in the HTML
|
|
standard.
|
|
|
|
TODO: This is specific to BeautifulSoup and its subclasses, but it's
|
|
used by __str__"""
|
|
return self.name in BeautifulSoup.SELF_CLOSING_TAGS
|
|
|
|
def append(self, tag):
|
|
"""Appends the given tag to the contents of this tag."""
|
|
self.contents.append(tag)
|
|
|
|
#Private methods
|
|
|
|
def _getAttrMap(self):
|
|
"""Initializes a map representation of this tag's attributes,
|
|
if not already initialized."""
|
|
if not getattr(self, 'attrMap'):
|
|
self.attrMap = {}
|
|
for (key, value) in self.attrs:
|
|
self.attrMap[key] = value
|
|
return self.attrMap
|
|
|
|
#Generator methods
|
|
def childGenerator(self):
|
|
for i in range(0, len(self.contents)):
|
|
yield self.contents[i]
|
|
raise StopIteration
|
|
|
|
def recursiveChildGenerator(self):
|
|
stack = [(self, 0)]
|
|
while stack:
|
|
tag, start = stack.pop()
|
|
if isinstance(tag, Tag):
|
|
for i in range(start, len(tag.contents)):
|
|
a = tag.contents[i]
|
|
yield a
|
|
if isinstance(a, Tag) and tag.contents:
|
|
if i < len(tag.contents) - 1:
|
|
stack.append((tag, i+1))
|
|
stack.append((a, 0))
|
|
break
|
|
raise StopIteration
|
|
|
|
|
|
def isList(l):
|
|
"""Convenience method that works with all 2.x versions of Python
|
|
to determine whether or not something is listlike."""
|
|
return hasattr(l, '__iter__') \
|
|
or (type(l) in (types.ListType, types.TupleType))
|
|
|
|
def buildTagMap(default, *args):
|
|
"""Turns a list of maps, lists, or scalars into a single map.
|
|
Used to build the SELF_CLOSING_TAGS and NESTABLE_TAGS maps out
|
|
of lists and partial maps."""
|
|
built = {}
|
|
for portion in args:
|
|
if hasattr(portion, 'items'):
|
|
#It's a map. Merge it.
|
|
for k,v in portion.items():
|
|
built[k] = v
|
|
elif isList(portion):
|
|
#It's a list. Map each item to the default.
|
|
for k in portion:
|
|
built[k] = default
|
|
else:
|
|
#It's a scalar. Map it to the default.
|
|
built[portion] = default
|
|
return built
|
|
|
|
class BeautifulStoneSoup(Tag, SGMLParser):
|
|
|
|
"""This class contains the basic parser and fetch code. It defines
|
|
a parser that knows nothing about tag behavior except for the
|
|
following:
|
|
|
|
You can't close a tag without closing all the tags it encloses.
|
|
That is, "<foo><bar></foo>" actually means
|
|
"<foo><bar></bar></foo>".
|
|
|
|
[Another possible explanation is "<foo><bar /></foo>", but since
|
|
this class defines no SELF_CLOSING_TAGS, it will never use that
|
|
explanation.]
|
|
|
|
This class is useful for parsing XML or made-up markup languages,
|
|
or when BeautifulSoup makes an assumption counter to what you were
|
|
expecting."""
|
|
|
|
SELF_CLOSING_TAGS = {}
|
|
NESTABLE_TAGS = {}
|
|
RESET_NESTING_TAGS = {}
|
|
QUOTE_TAGS = {}
|
|
|
|
#As a public service we will by default silently replace MS smart quotes
|
|
#and similar characters with their HTML or ASCII equivalents.
|
|
MS_CHARS = { '\x80' : '€',
|
|
'\x81' : ' ',
|
|
'\x82' : '‚',
|
|
'\x83' : 'ƒ',
|
|
'\x84' : '„',
|
|
'\x85' : '…',
|
|
'\x86' : '†',
|
|
'\x87' : '‡',
|
|
'\x88' : '⁁',
|
|
'\x89' : '%',
|
|
'\x8A' : 'Š',
|
|
'\x8B' : '<',
|
|
'\x8C' : 'Œ',
|
|
'\x8D' : '?',
|
|
'\x8E' : 'Z',
|
|
'\x8F' : '?',
|
|
'\x90' : '?',
|
|
'\x91' : '‘',
|
|
'\x92' : '’',
|
|
'\x93' : '“',
|
|
'\x94' : '”',
|
|
'\x95' : '•',
|
|
'\x96' : '–',
|
|
'\x97' : '—',
|
|
'\x98' : '˜',
|
|
'\x99' : '™',
|
|
'\x9a' : 'š',
|
|
'\x9b' : '>',
|
|
'\x9c' : 'œ',
|
|
'\x9d' : '?',
|
|
'\x9e' : 'z',
|
|
'\x9f' : 'Ÿ',}
|
|
|
|
PARSER_MASSAGE = [(re.compile('(<[^<>]*)/>'),
|
|
lambda(x):x.group(1) + ' />'),
|
|
(re.compile('<!\s+([^<>]*)>'),
|
|
lambda(x):'<!' + x.group(1) + '>'),
|
|
(re.compile("([\x80-\x9f])"),
|
|
lambda(x): BeautifulStoneSoup.MS_CHARS.get(x.group(1)))
|
|
]
|
|
|
|
ROOT_TAG_NAME = '[document]'
|
|
|
|
def __init__(self, text=None, avoidParserProblems=True,
|
|
initialTextIsEverything=True):
|
|
"""Initialize this as the 'root tag' and feed in any text to
|
|
the parser.
|
|
|
|
NOTE about avoidParserProblems: sgmllib will process most bad
|
|
HTML, and BeautifulSoup has tricks for dealing with some HTML
|
|
that kills sgmllib, but Beautiful Soup can nonetheless choke
|
|
or lose data if your data uses self-closing tags or
|
|
declarations incorrectly. By default, Beautiful Soup sanitizes
|
|
its input to avoid the vast majority of these problems. The
|
|
problems are relatively rare, even in bad HTML, so feel free
|
|
to pass in False to avoidParserProblems if they don't apply to
|
|
you, and you'll get better performance. The only reason I have
|
|
this turned on by default is so I don't get so many tech
|
|
support questions.
|
|
|
|
The two most common instances of invalid HTML that will choke
|
|
sgmllib are fixed by the default parser massage techniques:
|
|
|
|
<br/> (No space between name of closing tag and tag close)
|
|
<! --Comment--> (Extraneous whitespace in declaration)
|
|
|
|
You can pass in a custom list of (RE object, replace method)
|
|
tuples to get Beautiful Soup to scrub your input the way you
|
|
want."""
|
|
Tag.__init__(self, self.ROOT_TAG_NAME)
|
|
if avoidParserProblems \
|
|
and not isList(avoidParserProblems):
|
|
avoidParserProblems = self.PARSER_MASSAGE
|
|
self.avoidParserProblems = avoidParserProblems
|
|
SGMLParser.__init__(self)
|
|
self.quoteStack = []
|
|
self.hidden = 1
|
|
self.reset()
|
|
if hasattr(text, 'read'):
|
|
#It's a file-type object.
|
|
text = text.read()
|
|
if text:
|
|
self.feed(text)
|
|
if initialTextIsEverything:
|
|
self.done()
|
|
|
|
def __getattr__(self, methodName):
|
|
"""This method routes method call requests to either the SGMLParser
|
|
superclass or the Tag superclass, depending on the method name."""
|
|
if methodName.find('start_') == 0 or methodName.find('end_') == 0 \
|
|
or methodName.find('do_') == 0:
|
|
return SGMLParser.__getattr__(self, methodName)
|
|
elif methodName.find('__') != 0:
|
|
return Tag.__getattr__(self, methodName)
|
|
else:
|
|
raise AttributeError
|
|
|
|
def feed(self, text):
|
|
if self.avoidParserProblems:
|
|
for fix, m in self.avoidParserProblems:
|
|
text = fix.sub(m, text)
|
|
SGMLParser.feed(self, text)
|
|
|
|
def done(self):
|
|
"""Called when you're done parsing, so that the unclosed tags can be
|
|
correctly processed."""
|
|
self.endData() #NEW
|
|
while self.currentTag.name != self.ROOT_TAG_NAME:
|
|
self.popTag()
|
|
|
|
def reset(self):
|
|
SGMLParser.reset(self)
|
|
self.currentData = []
|
|
self.currentTag = None
|
|
self.tagStack = []
|
|
self.pushTag(self)
|
|
|
|
def popTag(self):
|
|
tag = self.tagStack.pop()
|
|
# Tags with just one string-owning child get the child as a
|
|
# 'string' property, so that soup.tag.string is shorthand for
|
|
# soup.tag.contents[0]
|
|
if len(self.currentTag.contents) == 1 and \
|
|
isinstance(self.currentTag.contents[0], NavigableText):
|
|
self.currentTag.string = self.currentTag.contents[0]
|
|
|
|
#print "Pop", tag.name
|
|
if self.tagStack:
|
|
self.currentTag = self.tagStack[-1]
|
|
return self.currentTag
|
|
|
|
def pushTag(self, tag):
|
|
#print "Push", tag.name
|
|
if self.currentTag:
|
|
self.currentTag.append(tag)
|
|
self.tagStack.append(tag)
|
|
self.currentTag = self.tagStack[-1]
|
|
|
|
def endData(self):
|
|
currentData = ''.join(self.currentData)
|
|
if currentData:
|
|
if not currentData.strip():
|
|
if '\n' in currentData:
|
|
currentData = '\n'
|
|
else:
|
|
currentData = ' '
|
|
c = NavigableString
|
|
if type(currentData) == types.UnicodeType:
|
|
c = NavigableUnicodeString
|
|
o = c(currentData)
|
|
o.setup(self.currentTag, self.previous)
|
|
if self.previous:
|
|
self.previous.next = o
|
|
self.previous = o
|
|
self.currentTag.contents.append(o)
|
|
self.currentData = []
|
|
|
|
def _popToTag(self, name, inclusivePop=True):
|
|
"""Pops the tag stack up to and including the most recent
|
|
instance of the given tag. If inclusivePop is false, pops the tag
|
|
stack up to but *not* including the most recent instqance of
|
|
the given tag."""
|
|
if name == self.ROOT_TAG_NAME:
|
|
return
|
|
|
|
numPops = 0
|
|
mostRecentTag = None
|
|
for i in range(len(self.tagStack)-1, 0, -1):
|
|
if name == self.tagStack[i].name:
|
|
numPops = len(self.tagStack)-i
|
|
break
|
|
if not inclusivePop:
|
|
numPops = numPops - 1
|
|
|
|
for i in range(0, numPops):
|
|
mostRecentTag = self.popTag()
|
|
return mostRecentTag
|
|
|
|
def _smartPop(self, name):
|
|
|
|
"""We need to pop up to the previous tag of this type, unless
|
|
one of this tag's nesting reset triggers comes between this
|
|
tag and the previous tag of this type, OR unless this tag is a
|
|
generic nesting trigger and another generic nesting trigger
|
|
comes between this tag and the previous tag of this type.
|
|
|
|
Examples:
|
|
<p>Foo<b>Bar<p> should pop to 'p', not 'b'.
|
|
<p>Foo<table>Bar<p> should pop to 'table', not 'p'.
|
|
<p>Foo<table><tr>Bar<p> should pop to 'tr', not 'p'.
|
|
<p>Foo<b>Bar<p> should pop to 'p', not 'b'.
|
|
|
|
<li><ul><li> *<li>* should pop to 'ul', not the first 'li'.
|
|
<tr><table><tr> *<tr>* should pop to 'table', not the first 'tr'
|
|
<td><tr><td> *<td>* should pop to 'tr', not the first 'td'
|
|
"""
|
|
|
|
nestingResetTriggers = self.NESTABLE_TAGS.get(name)
|
|
isNestable = nestingResetTriggers != None
|
|
isResetNesting = self.RESET_NESTING_TAGS.has_key(name)
|
|
popTo = None
|
|
inclusive = True
|
|
for i in range(len(self.tagStack)-1, 0, -1):
|
|
p = self.tagStack[i]
|
|
if (not p or p.name == name) and not isNestable:
|
|
#Non-nestable tags get popped to the top or to their
|
|
#last occurance.
|
|
popTo = name
|
|
break
|
|
if (nestingResetTriggers != None
|
|
and p.name in nestingResetTriggers) \
|
|
or (nestingResetTriggers == None and isResetNesting
|
|
and self.RESET_NESTING_TAGS.has_key(p.name)):
|
|
|
|
#If we encounter one of the nesting reset triggers
|
|
#peculiar to this tag, or we encounter another tag
|
|
#that causes nesting to reset, pop up to but not
|
|
#including that tag.
|
|
|
|
popTo = p.name
|
|
inclusive = False
|
|
break
|
|
p = p.parent
|
|
if popTo:
|
|
self._popToTag(popTo, inclusive)
|
|
|
|
def unknown_starttag(self, name, attrs, selfClosing=0):
|
|
#print "Start tag %s" % name
|
|
if self.quoteStack:
|
|
#This is not a real tag.
|
|
#print "<%s> is not real!" % name
|
|
attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs))
|
|
self.handle_data('<%s%s>' % (name, attrs))
|
|
return
|
|
self.endData()
|
|
if not name in self.SELF_CLOSING_TAGS and not selfClosing:
|
|
self._smartPop(name)
|
|
tag = Tag(name, attrs, self.currentTag, self.previous)
|
|
if self.previous:
|
|
self.previous.next = tag
|
|
self.previous = tag
|
|
self.pushTag(tag)
|
|
if selfClosing or name in self.SELF_CLOSING_TAGS:
|
|
self.popTag()
|
|
if name in self.QUOTE_TAGS:
|
|
#print "Beginning quote (%s)" % name
|
|
self.quoteStack.append(name)
|
|
self.literal = 1
|
|
|
|
def unknown_endtag(self, name):
|
|
if self.quoteStack and self.quoteStack[-1] != name:
|
|
#This is not a real end tag.
|
|
#print "</%s> is not real!" % name
|
|
self.handle_data('</%s>' % name)
|
|
return
|
|
self.endData()
|
|
self._popToTag(name)
|
|
if self.quoteStack and self.quoteStack[-1] == name:
|
|
self.quoteStack.pop()
|
|
self.literal = (len(self.quoteStack) > 0)
|
|
|
|
def handle_data(self, data):
|
|
self.currentData.append(data)
|
|
|
|
def handle_pi(self, text):
|
|
"Propagate processing instructions right through."
|
|
self.handle_data("<?%s>" % text)
|
|
|
|
def handle_comment(self, text):
|
|
"Propagate comments right through."
|
|
self.handle_data("<!--%s-->" % text)
|
|
|
|
def handle_charref(self, ref):
|
|
"Propagate char refs right through."
|
|
self.handle_data('&#%s;' % ref)
|
|
|
|
def handle_entityref(self, ref):
|
|
"Propagate entity refs right through."
|
|
self.handle_data('&%s;' % ref)
|
|
|
|
def handle_decl(self, data):
|
|
"Propagate DOCTYPEs and the like right through."
|
|
self.handle_data('<!%s>' % data)
|
|
|
|
def parse_declaration(self, i):
|
|
"""Treat a bogus SGML declaration as raw data. Treat a CDATA
|
|
declaration as regular data."""
|
|
j = None
|
|
if self.rawdata[i:i+9] == '<![CDATA[':
|
|
k = self.rawdata.find(']]>', i)
|
|
if k == -1:
|
|
k = len(self.rawdata)
|
|
self.handle_data(self.rawdata[i+9:k])
|
|
j = k+3
|
|
else:
|
|
try:
|
|
j = SGMLParser.parse_declaration(self, i)
|
|
except SGMLParseError:
|
|
toHandle = self.rawdata[i:]
|
|
self.handle_data(toHandle)
|
|
j = i + len(toHandle)
|
|
return j
|
|
|
|
class BeautifulSoup(BeautifulStoneSoup):
|
|
|
|
"""This parser knows the following facts about HTML:
|
|
|
|
* Some tags have no closing tag and should be interpreted as being
|
|
closed as soon as they are encountered.
|
|
|
|
* The text inside some tags (ie. 'script') may contain tags which
|
|
are not really part of the document and which should be parsed
|
|
as text, not tags. If you want to parse the text as tags, you can
|
|
always fetch it and parse it explicitly.
|
|
|
|
* Tag nesting rules:
|
|
|
|
Most tags can't be nested at all. For instance, the occurance of
|
|
a <p> tag should implicitly close the previous <p> tag.
|
|
|
|
<p>Para1<p>Para2
|
|
should be transformed into:
|
|
<p>Para1</p><p>Para2
|
|
|
|
Some tags can be nested arbitrarily. For instance, the occurance
|
|
of a <blockquote> tag should _not_ implicitly close the previous
|
|
<blockquote> tag.
|
|
|
|
Alice said: <blockquote>Bob said: <blockquote>Blah
|
|
should NOT be transformed into:
|
|
Alice said: <blockquote>Bob said: </blockquote><blockquote>Blah
|
|
|
|
Some tags can be nested, but the nesting is reset by the
|
|
interposition of other tags. For instance, a <tr> tag should
|
|
implicitly close the previous <tr> tag within the same <table>,
|
|
but not close a <tr> tag in another table.
|
|
|
|
<table><tr>Blah<tr>Blah
|
|
should be transformed into:
|
|
<table><tr>Blah</tr><tr>Blah
|
|
but,
|
|
<tr>Blah<table><tr>Blah
|
|
should NOT be transformed into
|
|
<tr>Blah<table></tr><tr>Blah
|
|
|
|
Differing assumptions about tag nesting rules are a major source
|
|
of problems with the BeautifulSoup class. If BeautifulSoup is not
|
|
treating as nestable a tag your page author treats as nestable,
|
|
try ICantBelieveItsBeautifulSoup before writing your own
|
|
subclass."""
|
|
|
|
SELF_CLOSING_TAGS = buildTagMap(None, ['br' , 'hr', 'input', 'img', 'meta',
|
|
'spacer', 'link', 'frame', 'base'])
|
|
|
|
QUOTE_TAGS = {'script': None}
|
|
|
|
#According to the HTML standard, each of these inline tags can
|
|
#contain another tag of the same type. Furthermore, it's common
|
|
#to actually use these tags this way.
|
|
NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup',
|
|
'center']
|
|
|
|
#According to the HTML standard, these block tags can contain
|
|
#another tag of the same type. Furthermore, it's common
|
|
#to actually use these tags this way.
|
|
NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del']
|
|
|
|
#Lists can contain other lists, but there are restrictions.
|
|
NESTABLE_LIST_TAGS = { 'ol' : [],
|
|
'ul' : [],
|
|
'li' : ['ul', 'ol'],
|
|
'dl' : [],
|
|
'dd' : ['dl'],
|
|
'dt' : ['dl'] }
|
|
|
|
#Tables can contain other tables, but there are restrictions.
|
|
NESTABLE_TABLE_TAGS = {'table' : [],
|
|
'tr' : ['table', 'tbody', 'tfoot', 'thead'],
|
|
'td' : ['tr'],
|
|
'th' : ['tr'],
|
|
}
|
|
|
|
NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre']
|
|
|
|
#If one of these tags is encountered, all tags up to the next tag of
|
|
#this type are popped.
|
|
RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript',
|
|
NON_NESTABLE_BLOCK_TAGS,
|
|
NESTABLE_LIST_TAGS,
|
|
NESTABLE_TABLE_TAGS)
|
|
|
|
NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS,
|
|
NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS)
|
|
|
|
class ICantBelieveItsBeautifulSoup(BeautifulSoup):
|
|
|
|
"""The BeautifulSoup class is oriented towards skipping over
|
|
common HTML errors like unclosed tags. However, sometimes it makes
|
|
errors of its own. For instance, consider this fragment:
|
|
|
|
<b>Foo<b>Bar</b></b>
|
|
|
|
This is perfectly valid (if bizarre) HTML. However, the
|
|
BeautifulSoup class will implicitly close the first b tag when it
|
|
encounters the second 'b'. It will think the author wrote
|
|
"<b>Foo<b>Bar", and didn't close the first 'b' tag, because
|
|
there's no real-world reason to bold something that's already
|
|
bold. When it encounters '</b></b>' it will close two more 'b'
|
|
tags, for a grand total of three tags closed instead of two. This
|
|
can throw off the rest of your document structure. The same is
|
|
true of a number of other tags, listed below.
|
|
|
|
It's much more common for someone to forget to close (eg.) a 'b'
|
|
tag than to actually use nested 'b' tags, and the BeautifulSoup
|
|
class handles the common case. This class handles the
|
|
not-co-common case: where you can't believe someone wrote what
|
|
they did, but it's valid HTML and BeautifulSoup screwed up by
|
|
assuming it wouldn't be.
|
|
|
|
If this doesn't do what you need, try subclassing this class or
|
|
BeautifulSoup, and providing your own list of NESTABLE_TAGS."""
|
|
|
|
I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \
|
|
['em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong',
|
|
'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b',
|
|
'big']
|
|
|
|
I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript']
|
|
|
|
NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS,
|
|
I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS,
|
|
I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS)
|
|
|
|
class BeautifulSOAP(BeautifulStoneSoup):
|
|
"""This class will push a tag with only a single string child into
|
|
the tag's parent as an attribute. The attribute's name is the tag
|
|
name, and the value is the string child. An example should give
|
|
the flavor of the change:
|
|
|
|
<foo><bar>baz</bar></foo>
|
|
=>
|
|
<foo bar="baz"><bar>baz</bar></foo>
|
|
|
|
You can then access fooTag['bar'] instead of fooTag.barTag.string.
|
|
|
|
This is, of course, useful for scraping structures that tend to
|
|
use subelements instead of attributes, such as SOAP messages. Note
|
|
that it modifies its input, so don't print the modified version
|
|
out.
|
|
|
|
I'm not sure how many people really want to use this class; let me
|
|
know if you do. Mainly I like the name."""
|
|
|
|
def popTag(self):
|
|
if len(self.tagStack) > 1:
|
|
tag = self.tagStack[-1]
|
|
parent = self.tagStack[-2]
|
|
parent._getAttrMap()
|
|
if (isinstance(tag, Tag) and len(tag.contents) == 1 and
|
|
isinstance(tag.contents[0], NavigableText) and
|
|
not parent.attrMap.has_key(tag.name)):
|
|
parent[tag.name] = tag.contents[0]
|
|
BeautifulStoneSoup.popTag(self)
|
|
|
|
#Enterprise class names! It has come to our attention that some people
|
|
#think the names of the Beautiful Soup parser classes are too silly
|
|
#and "unprofessional" for use in enterprise screen-scraping. We feel
|
|
#your pain! For such-minded folk, the Beautiful Soup Consortium And
|
|
#All-Night Kosher Bakery recommends renaming this file to
|
|
#"RobustParser.py" (or, in cases of extreme enterprisitude,
|
|
#"RobustParserBeanInterface.class") and using the following
|
|
#enterprise-friendly class aliases:
|
|
class RobustXMLParser(BeautifulStoneSoup):
|
|
pass
|
|
class RobustHTMLParser(BeautifulSoup):
|
|
pass
|
|
class RobustWackAssHTMLParser(ICantBelieveItsBeautifulSoup):
|
|
pass
|
|
class SimplifyingSOAPParser(BeautifulSOAP):
|
|
pass
|
|
|
|
###
|
|
|
|
|
|
#By default, act as an HTML pretty-printer.
|
|
if __name__ == '__main__':
|
|
import sys
|
|
soup = BeautifulStoneSoup(sys.stdin.read())
|
|
print soup.prettify()
|