1: """Beautiful Soup
2: Elixir and Tonic
3: "The Screen-Scraper's Friend"
4: http://www.crummy.com/software/BeautifulSoup/
5:
6: Beautiful Soup parses a (possibly invalid) XML or HTML document into a
7: tree representation. It provides methods and Pythonic idioms that make
8: it easy to navigate, search, and modify the tree.
9:
10: A well-formed XML/HTML document yields a well-formed data
11: structure. An ill-formed XML/HTML document yields a correspondingly
12: ill-formed data structure. If your document is only locally
13: well-formed, you can use this library to find and process the
14: well-formed part of it.
15:
16: Beautiful Soup works with Python 2.2 and up. It has no external
17: dependencies, but you'll have more success at converting data to UTF-8
18: if you also install these three packages:
19:
20: * chardet, for auto-detecting character encodings
21: http://chardet.feedparser.org/
22: * cjkcodecs and iconv_codec, which add more encodings to the ones supported
23: by stock Python.
24: http://cjkpython.i18n.org/
25:
26: Beautiful Soup defines classes for two main parsing strategies:
27:
28: * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific
29: language that kind of looks like XML.
30:
31: * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid
32: or invalid. This class has web browser-like heuristics for
33: obtaining a sensible parse tree in the face of common HTML errors.
34:
35: Beautiful Soup also defines a class (UnicodeDammit) for autodetecting
36: the encoding of an HTML or XML document, and converting it to
37: Unicode. Much of this code is taken from Mark Pilgrim's Universal Feed Parser.
38:
39: For more than you ever wanted to know about Beautiful Soup, see the
40: documentation:
41: http://www.crummy.com/software/BeautifulSoup/documentation.html
42:
43: Here, have some legalese:
44:
45: Copyright (c) 2004-2010, Leonard Richardson
46:
47: All rights reserved.
48:
49: Redistribution and use in source and binary forms, with or without
50: modification, are permitted provided that the following conditions are
51: met:
52:
53: * Redistributions of source code must retain the above copyright
54: notice, this list of conditions and the following disclaimer.
55:
56: * Redistributions in binary form must reproduce the above
57: copyright notice, this list of conditions and the following
58: disclaimer in the documentation and/or other materials provided
59: with the distribution.
60:
61: * Neither the name of the the Beautiful Soup Consortium and All
62: Night Kosher Bakery nor the names of its contributors may be
63: used to endorse or promote products derived from this software
64: without specific prior written permission.
65:
66: THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
67: "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
68: LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
69: A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
70: CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
71: EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
72: PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
73: PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
74: LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
75: NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
76: SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT.
77:
78: """
79: from __future__ import generators
80:
81: __author__ = "Leonard Richardson (leonardr@segfault.org)"
82: __version__ = "3.2.0"
83: __copyright__ = "Copyright (c) 2004-2010 Leonard Richardson"
84: __license__ = "New-style BSD"
85:
86: from sgmllib import SGMLParser, SGMLParseError
87: import codecs
88: import markupbase
89: import types
90: import re
91: import sgmllib
92: try:
93: from htmlentitydefs import name2codepoint
94: except ImportError:
95: name2codepoint = {}
96: try:
97: set
98: except NameError:
99: from sets import Set as set
100:
101: #These hacks make Beautiful Soup able to parse XML with namespaces
102: sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*')
103: markupbase._declname_match = re.compile(r'[a-zA-Z][-_.:a-zA-Z0-9]*\s*').match
104:
105: DEFAULT_OUTPUT_ENCODING = "utf-8"
106:
107: def _match_css_class(str):
108: """Build a RE to match the given CSS class."""
109: return re.compile(r"(^|.*\s)%s($|\s)" % str)
110:
111: # First, the classes that represent markup elements.
112:
113: class PageElement(object):
114: """Contains the navigational information for some part of the page
115: (either a tag or a piece of text)"""
116:
117: def setup(self, parent=None, previous=None):
118: """Sets up the initial relations between this element and
119: other elements."""
120: self.parent = parent
121: self.previous = previous
122: self.next = None
123: self.previousSibling = None
124: self.nextSibling = None
125: if self.parent and self.parent.contents:
126: self.previousSibling = self.parent.contents[-1]
127: self.previousSibling.nextSibling = self
128:
129: def replaceWith(self, replaceWith):
130: oldParent = self.parent
131: myIndex = self.parent.index(self)
132: if hasattr(replaceWith, "parent")\
133: and replaceWith.parent is self.parent:
134: # We're replacing this element with one of its siblings.
135: index = replaceWith.parent.index(replaceWith)
136: if index and index < myIndex:
137: # Furthermore, it comes before this element. That
138: # means that when we extract it, the index of this
139: # element will change.
140: myIndex = myIndex - 1
141: self.extract()
142: oldParent.insert(myIndex, replaceWith)
143:
144: def replaceWithChildren(self):
145: myParent = self.parent
146: myIndex = self.parent.index(self)
147: self.extract()
148: reversedChildren = list(self.contents)
149: reversedChildren.reverse()
150: for child in reversedChildren:
151: myParent.insert(myIndex, child)
152:
153: def extract(self):
154: """Destructively rips this element out of the tree."""
155: if self.parent:
156: try:
157: del self.parent.contents[self.parent.index(self)]
158: except ValueError:
159: pass
160:
161: #Find the two elements that would be next to each other if
162: #this element (and any children) hadn't been parsed. Connect
163: #the two.
164: lastChild = self._lastRecursiveChild()
165: nextElement = lastChild.next
166:
167: if self.previous:
168: self.previous.next = nextElement
169: if nextElement:
170: nextElement.previous = self.previous
171: self.previous = None
172: lastChild.next = None
173:
174: self.parent = None
175: if self.previousSibling:
176: self.previousSibling.nextSibling = self.nextSibling
177: if self.nextSibling:
178: self.nextSibling.previousSibling = self.previousSibling
179: self.previousSibling = self.nextSibling = None
180: return self
181:
182: def _lastRecursiveChild(self):
183: "Finds the last element beneath this object to be parsed."
184: lastChild = self
185: while hasattr(lastChild, 'contents') and lastChild.contents:
186: lastChild = lastChild.contents[-1]
187: return lastChild
188:
189: def insert(self, position, newChild):
190: if isinstance(newChild, basestring) \
191: and not isinstance(newChild, NavigableString):
192: newChild = NavigableString(newChild)
193:
194: position = min(position, len(self.contents))
195: if hasattr(newChild, 'parent') and newChild.parent is not None:
196: # We're 'inserting' an element that's already one
197: # of this object's children.
198: if newChild.parent is self:
199: index = self.index(newChild)
200: if index > position:
201: # Furthermore we're moving it further down the
202: # list of this object's children. That means that
203: # when we extract this element, our target index
204: # will jump down one.
205: position = position - 1
206: newChild.extract()
207:
208: newChild.parent = self
209: previousChild = None
210: if position == 0:
211: newChild.previousSibling = None
212: newChild.previous = self
213: else:
214: previousChild = self.contents[position-1]
215: newChild.previousSibling = previousChild
216: newChild.previousSibling.nextSibling = newChild
217: newChild.previous = previousChild._lastRecursiveChild()
218: if newChild.previous:
219: newChild.previous.next = newChild
220:
221: newChildsLastElement = newChild._lastRecursiveChild()
222:
223: if position >= len(self.contents):
224: newChild.nextSibling = None
225:
226: parent = self
227: parentsNextSibling = None
228: while not parentsNextSibling:
229: parentsNextSibling = parent.nextSibling
230: parent = parent.parent
231: if not parent: # This is the last element in the document.
232: break
233: if parentsNextSibling:
234: newChildsLastElement.next = parentsNextSibling
235: else:
236: newChildsLastElement.next = None
237: else:
238: nextChild = self.contents[position]
239: newChild.nextSibling = nextChild
240: if newChild.nextSibling:
241: newChild.nextSibling.previousSibling = newChild
242: newChildsLastElement.next = nextChild
243:
244: if newChildsLastElement.next:
245: newChildsLastElement.next.previous = newChildsLastElement
246: self.contents.insert(position, newChild)
247:
248: def append(self, tag):
249: """Appends the given tag to the contents of this tag."""
250: self.insert(len(self.contents), tag)
251:
252: def findNext(self, name=None, attrs={}, text=None, **kwargs):
253: """Returns the first item that matches the given criteria and
254: appears after this Tag in the document."""
255: return self._findOne(self.findAllNext, name, attrs, text, **kwargs)
256:
257: def findAllNext(self, name=None, attrs={}, text=None, limit=None,
258: **kwargs):
259: """Returns all items that match the given criteria and appear
260: after this Tag in the document."""
261: return self._findAll(name, attrs, text, limit, self.nextGenerator,
262: **kwargs)
263:
264: def findNextSibling(self, name=None, attrs={}, text=None, **kwargs):
265: """Returns the closest sibling to this Tag that matches the
266: given criteria and appears after this Tag in the document."""
267: return self._findOne(self.findNextSiblings, name, attrs, text,
268: **kwargs)
269:
270: def findNextSiblings(self, name=None, attrs={}, text=None, limit=None,
271: **kwargs):
272: """Returns the siblings of this Tag that match the given
273: criteria and appear after this Tag in the document."""
274: return self._findAll(name, attrs, text, limit,
275: self.nextSiblingGenerator, **kwargs)
276: fetchNextSiblings = findNextSiblings # Compatibility with pre-3.x
277:
278: def findPrevious(self, name=None, attrs={}, text=None, **kwargs):
279: """Returns the first item that matches the given criteria and
280: appears before this Tag in the document."""
281: return self._findOne(self.findAllPrevious, name, attrs, text, **kwargs)
282:
283: def findAllPrevious(self, name=None, attrs={}, text=None, limit=None,
284: **kwargs):
285: """Returns all items that match the given criteria and appear
286: before this Tag in the document."""
287: return self._findAll(name, attrs, text, limit, self.previousGenerator,
288: **kwargs)
289: fetchPrevious = findAllPrevious # Compatibility with pre-3.x
290:
291: def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs):
292: """Returns the closest sibling to this Tag that matches the
293: given criteria and appears before this Tag in the document."""
294: return self._findOne(self.findPreviousSiblings, name, attrs, text,
295: **kwargs)
296:
297: def findPreviousSiblings(self, name=None, attrs={}, text=None,
298: limit=None, **kwargs):
299: """Returns the siblings of this Tag that match the given
300: criteria and appear before this Tag in the document."""
301: return self._findAll(name, attrs, text, limit,
302: self.previousSiblingGenerator, **kwargs)
303: fetchPreviousSiblings = findPreviousSiblings # Compatibility with pre-3.x
304:
305: def findParent(self, name=None, attrs={}, **kwargs):
306: """Returns the closest parent of this Tag that matches the given
307: criteria."""
308: # NOTE: We can't use _findOne because findParents takes a different
309: # set of arguments.
310: r = None
311: l = self.findParents(name, attrs, 1)
312: if l:
313: r = l[0]
314: return r
315:
316: def findParents(self, name=None, attrs={}, limit=None, **kwargs):
317: """Returns the parents of this Tag that match the given
318: criteria."""
319:
320: return self._findAll(name, attrs, None, limit, self.parentGenerator,
321: **kwargs)
322: fetchParents = findParents # Compatibility with pre-3.x
323:
324: #These methods do the real heavy lifting.
325:
326: def _findOne(self, method, name, attrs, text, **kwargs):
327: r = None
328: l = method(name, attrs, text, 1, **kwargs)
329: if l:
330: r = l[0]
331: return r
332:
333: def _findAll(self, name, attrs, text, limit, generator, **kwargs):
334: "Iterates over a generator looking for things that match."
335:
336: if isinstance(name, SoupStrainer):
337: strainer = name
338: # (Possibly) special case some findAll*(...) searches
339: elif text is None and not limit and not attrs and not kwargs:
340: # findAll*(True)
341: if name is True:
342: return [element for element in generator()
343: if isinstance(element, Tag)]
344: # findAll*('tag-name')
345: elif isinstance(name, basestring):
346: return [element for element in generator()
347: if isinstance(element, Tag) and
348: element.name == name]
349: else:
350: strainer = SoupStrainer(name, attrs, text, **kwargs)
351: # Build a SoupStrainer
352: else:
353: strainer = SoupStrainer(name, attrs, text, **kwargs)
354: results = ResultSet(strainer)
355: g = generator()
356: while True:
357: try:
358: i = g.next()
359: except StopIteration:
360: break
361: if i:
362: found = strainer.search(i)
363: if found:
364: results.append(found)
365: if limit and len(results) >= limit:
366: break
367: return results
368:
369: #These Generators can be used to navigate starting from both
370: #NavigableStrings and Tags.
371: def nextGenerator(self):
372: i = self
373: while i is not None:
374: i = i.next
375: yield i
376:
377: def nextSiblingGenerator(self):
378: i = self
379: while i is not None:
380: i = i.nextSibling
381: yield i
382:
383: def previousGenerator(self):
384: i = self
385: while i is not None:
386: i = i.previous
387: yield i
388:
389: def previousSiblingGenerator(self):
390: i = self
391: while i is not None:
392: i = i.previousSibling
393: yield i
394:
395: def parentGenerator(self):
396: i = self
397: while i is not None:
398: i = i.parent
399: yield i
400:
401: # Utility methods
402: def substituteEncoding(self, str, encoding=None):
403: encoding = encoding or "utf-8"
404: return str.replace("%SOUP-ENCODING%", encoding)
405:
406: def toEncoding(self, s, encoding=None):
407: """Encodes an object to a string in some encoding, or to Unicode.
408: ."""
409: if isinstance(s, unicode):
410: if encoding:
411: s = s.encode(encoding)
412: elif isinstance(s, str):
413: if encoding:
414: s = s.encode(encoding)
415: else:
416: s = unicode(s)
417: else:
418: if encoding:
419: s = self.toEncoding(str(s), encoding)
420: else:
421: s = unicode(s)
422: return s
423:
424: class NavigableString(unicode, PageElement):
425:
426: def __new__(cls, value):
427: """Create a new NavigableString.
428:
429: When unpickling a NavigableString, this method is called with
430: the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be
431: passed in to the superclass's __new__ or the superclass won't know
432: how to handle non-ASCII characters.
433: """
434: if isinstance(value, unicode):
435: return unicode.__new__(cls, value)
436: return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING)
437:
438: def __getnewargs__(self):
439: return (NavigableString.__str__(self),)
440:
441: def __getattr__(self, attr):
442: """text.string gives you text. This is for backwards
443: compatibility for Navigable*String, but for CData* it lets you
444: get the string without the CData wrapper."""
445: if attr == 'string':
446: return self
447: else:
448: raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr)
449:
450: def __unicode__(self):
451: return str(self).decode(DEFAULT_OUTPUT_ENCODING)
452:
453: def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
454: if encoding:
455: return self.encode(encoding)
456: else:
457: return self
458:
459: class CData(NavigableString):
460:
461: def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
462: return "<![CDATA[%s]]>" % NavigableString.__str__(self, encoding)
463:
464: class ProcessingInstruction(NavigableString):
465: def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
466: output = self
467: if "%SOUP-ENCODING%" in output:
468: output = self.substituteEncoding(output, encoding)
469: return "<?%s?>" % self.toEncoding(output, encoding)
470:
471: class Comment(NavigableString):
472: def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
473: return "<!--%s-->" % NavigableString.__str__(self, encoding)
474:
475: class Declaration(NavigableString):
476: def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
477: return "<!%s>" % NavigableString.__str__(self, encoding)
478:
479: class Tag(PageElement):
480:
481: """Represents a found HTML tag with its attributes and contents."""
482:
483: def _invert(h):
484: "Cheap function to invert a hash."
485: i = {}
486: for k,v in h.items():
487: i[v] = k
488: return i
489:
490: XML_ENTITIES_TO_SPECIAL_CHARS = { "apos" : "'",
491: "quot" : '"',
492: "amp" : "&",
493: "lt" : "<",
494: "gt" : ">" }
495:
496: XML_SPECIAL_CHARS_TO_ENTITIES = _invert(XML_ENTITIES_TO_SPECIAL_CHARS)
497:
498: def _convertEntities(self, match):
499: """Used in a call to re.sub to replace HTML, XML, and numeric
500: entities with the appropriate Unicode characters. If HTML
501: entities are being converted, any unrecognized entities are
502: escaped."""
503: x = match.group(1)
504: if self.convertHTMLEntities and x in name2codepoint:
505: return unichr(name2codepoint[x])
506: elif x in self.XML_ENTITIES_TO_SPECIAL_CHARS:
507: if self.convertXMLEntities:
508: return self.XML_ENTITIES_TO_SPECIAL_CHARS[x]
509: else:
510: return u'&%s;' % x
511: elif len(x) > 0 and x[0] == '#':
512: # Handle numeric entities
513: if len(x) > 1 and x[1] == 'x':
514: return unichr(int(x[2:], 16))
515: else:
516: return unichr(int(x[1:]))
517:
518: elif self.escapeUnrecognizedEntities:
519: return u'&%s;' % x
520: else:
521: return u'&%s;' % x
522:
523: def __init__(self, parser, name, attrs=None, parent=None,
524: previous=None):
525: "Basic constructor."
526:
527: # We don't actually store the parser object: that lets extracted
528: # chunks be garbage-collected
529: self.parserClass = parser.__class__
530: self.isSelfClosing = parser.isSelfClosingTag(name)
531: self.name = name
532: if attrs is None:
533: attrs = []
534: elif isinstance(attrs, dict):
535: attrs = attrs.items()
536: self.attrs = attrs
537: self.contents = []
538: self.setup(parent, previous)
539: self.hidden = False
540: self.containsSubstitutions = False
541: self.convertHTMLEntities = parser.convertHTMLEntities
542: self.convertXMLEntities = parser.convertXMLEntities
543: self.escapeUnrecognizedEntities = parser.escapeUnrecognizedEntities
544:
545: # Convert any HTML, XML, or numeric entities in the attribute values.
546: convert = lambda(k, val): (k,
547: re.sub("&(#\d+|#x[0-9a-fA-F]+|\w+);",
548: self._convertEntities,
549: val))
550: self.attrs = map(convert, self.attrs)
551:
552: def getString(self):
553: if (len(self.contents) == 1
554: and isinstance(self.contents[0], NavigableString)):
555: return self.contents[0]
556:
557: def setString(self, string):
558: """Replace the contents of the tag with a string"""
559: self.clear()
560: self.append(string)
561:
562: string = property(getString, setString)
563:
564: def getText(self, separator=u""):
565: if not len(self.contents):
566: return u""
567: stopNode = self._lastRecursiveChild().next
568: strings = []
569: current = self.contents[0]
570: while current is not stopNode:
571: if isinstance(current, NavigableString):
572: strings.append(current.strip())
573: current = current.next
574: return separator.join(strings)
575:
576: text = property(getText)
577:
578: def get(self, key, default=None):
579: """Returns the value of the 'key' attribute for the tag, or
580: the value given for 'default' if it doesn't have that
581: attribute."""
582: return self._getAttrMap().get(key, default)
583:
584: def clear(self):
585: """Extract all children."""
586: for child in self.contents[:]:
587: child.extract()
588:
589: def index(self, element):
590: for i, child in enumerate(self.contents):
591: if child is element:
592: return i
593: raise ValueError("Tag.index: element not in tag")
594:
595: def has_key(self, key):
596: return self._getAttrMap().has_key(key)
597:
598: def __getitem__(self, key):
599: """tag[key] returns the value of the 'key' attribute for the tag,
600: and throws an exception if it's not there."""
601: return self._getAttrMap()[key]
602:
603: def __iter__(self):
604: "Iterating over a tag iterates over its contents."
605: return iter(self.contents)
606:
607: def __len__(self):
608: "The length of a tag is the length of its list of contents."
609: return len(self.contents)
610:
611: def __contains__(self, x):
612: return x in self.contents
613:
614: def __nonzero__(self):
615: "A tag is non-None even if it has no contents."
616: return True
617:
618: def __setitem__(self, key, value):
619: """Setting tag[key] sets the value of the 'key' attribute for the
620: tag."""
621: self._getAttrMap()
622: self.attrMap[key] = value
623: found = False
624: for i in range(0, len(self.attrs)):
625: if self.attrs[i][0] == key:
626: self.attrs[i] = (key, value)
627: found = True
628: if not found:
629: self.attrs.append((key, value))
630: self._getAttrMap()[key] = value
631:
632: def __delitem__(self, key):
633: "Deleting tag[key] deletes all 'key' attributes for the tag."
634: for item in self.attrs:
635: if item[0] == key:
636: self.attrs.remove(item)
637: #We don't break because bad HTML can define the same
638: #attribute multiple times.
639: self._getAttrMap()
640: if self.attrMap.has_key(key):
641: del self.attrMap[key]
642:
643: def __call__(self, *args, **kwargs):
644: """Calling a tag like a function is the same as calling its
645: findAll() method. Eg. tag('a') returns a list of all the A tags
646: found within this tag."""
647: return apply(self.findAll, args, kwargs)
648:
649: def __getattr__(self, tag):
650: #print "Getattr %s.%s" % (self.__class__, tag)
651: if len(tag) > 3 and tag.rfind('Tag') == len(tag)-3:
652: return self.find(tag[:-3])
653: elif tag.find('__') != 0:
654: return self.find(tag)
655: raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__, tag)
656:
657: def __eq__(self, other):
658: """Returns true iff this tag has the same name, the same attributes,
659: and the same contents (recursively) as the given tag.
660:
661: NOTE: right now this will return false if two tags have the
662: same attributes in a different order. Should this be fixed?"""
663: if other is self:
664: return True
665: if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other):
666: return False
667: for i in range(0, len(self.contents)):
668: if self.contents[i] != other.contents[i]:
669: return False
670: return True
671:
672: def __ne__(self, other):
673: """Returns true iff this tag is not identical to the other tag,
674: as defined in __eq__."""
675: return not self == other
676:
677: def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING):
678: """Renders this tag as a string."""
679: return self.__str__(encoding)
680:
681: def __unicode__(self):
682: return self.__str__(None)
683:
684: BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
685: + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
686: + ")")
687:
688: def _sub_entity(self, x):
689: """Used with a regular expression to substitute the
690: appropriate XML entity for an XML special character."""
691: return "&" + self.XML_SPECIAL_CHARS_TO_ENTITIES[x.group(0)[0]] + ";"
692:
693: def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING,
694: prettyPrint=False, indentLevel=0):
695: """Returns a string or Unicode representation of this tag and
696: its contents. To get Unicode, pass None for encoding.
697:
698: NOTE: since Python's HTML parser consumes whitespace, this
699: method is not certain to reproduce the whitespace present in
700: the original string."""
701:
702: encodedName = self.toEncoding(self.name, encoding)
703:
704: attrs = []
705: if self.attrs:
706: for key, val in self.attrs:
707: fmt = '%s="%s"'
708: if isinstance(val, basestring):
709: if self.containsSubstitutions and '%SOUP-ENCODING%' in val:
710: val = self.substituteEncoding(val, encoding)
711:
712: # The attribute value either:
713: #
714: # * Contains no embedded double quotes or single quotes.
715: # No problem: we enclose it in double quotes.
716: # * Contains embedded single quotes. No problem:
717: # double quotes work here too.
718: # * Contains embedded double quotes. No problem:
719: # we enclose it in single quotes.
720: # * Embeds both single _and_ double quotes. This
721: # can't happen naturally, but it can happen if
722: # you modify an attribute value after parsing
723: # the document. Now we have a bit of a
724: # problem. We solve it by enclosing the
725: # attribute in single quotes, and escaping any
726: # embedded single quotes to XML entities.
727: if '"' in val:
728: fmt = "%s='%s'"
729: if "'" in val:
730: # TODO: replace with apos when
731: # appropriate.
732: val = val.replace("'", "&squot;")
733:
734: # Now we're okay w/r/t quotes. But the attribute
735: # value might also contain angle brackets, or
736: # ampersands that aren't part of entities. We need
737: # to escape those to XML entities too.
738: val = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, val)
739:
740: attrs.append(fmt % (self.toEncoding(key, encoding),
741: self.toEncoding(val, encoding)))
742: close = ''
743: closeTag = ''
744: if self.isSelfClosing:
745: close = ' /'
746: else:
747: closeTag = '</%s>' % encodedName
748:
749: indentTag, indentContents = 0, 0
750: if prettyPrint:
751: indentTag = indentLevel
752: space = (' ' * (indentTag-1))
753: indentContents = indentTag + 1
754: contents = self.renderContents(encoding, prettyPrint, indentContents)
755: if self.hidden:
756: s = contents
757: else:
758: s = []
759: attributeString = ''
760: if attrs:
761: attributeString = ' ' + ' '.join(attrs)
762: if prettyPrint:
763: s.append(space)
764: s.append('<%s%s%s>' % (encodedName, attributeString, close))
765: if prettyPrint:
766: s.append("\n")
767: s.append(contents)
768: if prettyPrint and contents and contents[-1] != "\n":
769: s.append("\n")
770: if prettyPrint and closeTag:
771: s.append(space)
772: s.append(closeTag)
773: if prettyPrint and closeTag and self.nextSibling:
774: s.append("\n")
775: s = ''.join(s)
776: return s
777:
778: def decompose(self):
779: """Recursively destroys the contents of this tree."""
780: self.extract()
781: if len(self.contents) == 0:
782: return
783: current = self.contents[0]
784: while current is not None:
785: next = current.next
786: if isinstance(current, Tag):
787: del current.contents[:]
788: current.parent = None
789: current.previous = None
790: current.previousSibling = None
791: current.next = None
792: current.nextSibling = None
793: current = next
794:
795: def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING):
796: return self.__str__(encoding, True)
797:
798: def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING,
799: prettyPrint=False, indentLevel=0):
800: """Renders the contents of this tag as a string in the given
801: encoding. If encoding is None, returns a Unicode string.."""
802: s=[]
803: for c in self:
804: text = None
805: if isinstance(c, NavigableString):
806: text = c.__str__(encoding)
807: elif isinstance(c, Tag):
808: s.append(c.__str__(encoding, prettyPrint, indentLevel))
809: if text and prettyPrint:
810: text = text.strip()
811: if text:
812: if prettyPrint:
813: s.append(" " * (indentLevel-1))
814: s.append(text)
815: if prettyPrint:
816: s.append("\n")
817: return ''.join(s)
818:
819: #Soup methods
820:
821: def find(self, name=None, attrs={}, recursive=True, text=None,
822: **kwargs):
823: """Return only the first child of this Tag matching the given
824: criteria."""
825: r = None
826: l = self.findAll(name, attrs, recursive, text, 1, **kwargs)
827: if l:
828: r = l[0]
829: return r
830: findChild = find
831:
832: def findAll(self, name=None, attrs={}, recursive=True, text=None,
833: limit=None, **kwargs):
834: """Extracts a list of Tag objects that match the given
835: criteria. You can specify the name of the Tag and any
836: attributes you want the Tag to have.
837:
838: The value of a key-value pair in the 'attrs' map can be a
839: string, a list of strings, a regular expression object, or a
840: callable that takes a string and returns whether or not the
841: string matches for some custom definition of 'matches'. The
842: same is true of the tag name."""
843: generator = self.recursiveChildGenerator
844: if not recursive:
845: generator = self.childGenerator
846: return self._findAll(name, attrs, text, limit, generator, **kwargs)
847: findChildren = findAll
848:
849: # Pre-3.x compatibility methods
850: first = find
851: fetch = findAll
852:
853: def fetchText(self, text=None, recursive=True, limit=None):
854: return self.findAll(text=text, recursive=recursive, limit=limit)
855:
856: def firstText(self, text=None, recursive=True):
857: return self.find(text=text, recursive=recursive)
858:
859: #Private methods
860:
861: def _getAttrMap(self):
862: """Initializes a map representation of this tag's attributes,
863: if not already initialized."""
864: if not getattr(self, 'attrMap'):
865: self.attrMap = {}
866: for (key, value) in self.attrs:
867: self.attrMap[key] = value
868: return self.attrMap
869:
870: #Generator methods
871: def childGenerator(self):
872: # Just use the iterator from the contents
873: return iter(self.contents)
874:
875: def recursiveChildGenerator(self):
876: if not len(self.contents):
877: raise StopIteration
878: stopNode = self._lastRecursiveChild().next
879: current = self.contents[0]
880: while current is not stopNode:
881: yield current
882: current = current.next
883:
884:
885: # Next, a couple classes to represent queries and their results.
886: class SoupStrainer:
887: """Encapsulates a number of ways of matching a markup element (tag or
888: text)."""
889:
890: def __init__(self, name=None, attrs={}, text=None, **kwargs):
891: self.name = name
892: if isinstance(attrs, basestring):
893: kwargs['class'] = _match_css_class(attrs)
894: attrs = None
895: if kwargs:
896: if attrs:
897: attrs = attrs.copy()
898: attrs.update(kwargs)
899: else:
900: attrs = kwargs
901: self.attrs = attrs
902: self.text = text
903:
904: def __str__(self):
905: if self.text:
906: return self.text
907: else:
908: return "%s|%s" % (self.name, self.attrs)
909:
910: def searchTag(self, markupName=None, markupAttrs={}):
911: found = None
912: markup = None
913: if isinstance(markupName, Tag):
914: markup = markupName
915: markupAttrs = markup
916: callFunctionWithTagData = callable(self.name) \
917: and not isinstance(markupName, Tag)
918:
919: if (not self.name) \
920: or callFunctionWithTagData \
921: or (markup and self._matches(markup, self.name)) \
922: or (not markup and self._matches(markupName, self.name)):
923: if callFunctionWithTagData:
924: match = self.name(markupName, markupAttrs)
925: else:
926: match = True
927: markupAttrMap = None
928: for attr, matchAgainst in self.attrs.items():
929: if not markupAttrMap:
930: if hasattr(markupAttrs, 'get'):
931: markupAttrMap = markupAttrs
932: else:
933: markupAttrMap = {}
934: for k,v in markupAttrs:
935: markupAttrMap[k] = v
936: attrValue = markupAttrMap.get(attr)
937: if not self._matches(attrValue, matchAgainst):
938: match = False
939: break
940: if match:
941: if markup:
942: found = markup
943: else:
944: found = markupName
945: return found
946:
947: def search(self, markup):
948: #print 'looking for %s in %s' % (self, markup)
949: found = None
950: # If given a list of items, scan it for a text element that
951: # matches.
952: if hasattr(markup, "__iter__") \
953: and not isinstance(markup, Tag):
954: for element in markup:
955: if isinstance(element, NavigableString) \
956: and self.search(element):
957: found = element
958: break
959: # If it's a Tag, make sure its name or attributes match.
960: # Don't bother with Tags if we're searching for text.
961: elif isinstance(markup, Tag):
962: if not self.text:
963: found = self.searchTag(markup)
964: # If it's text, make sure the text matches.
965: elif isinstance(markup, NavigableString) or \
966: isinstance(markup, basestring):
967: if self._matches(markup, self.text):
968: found = markup
969: else:
970: raise Exception, "I don't know how to match against a %s" \
971: % markup.__class__
972: return found
973:
974: def _matches(self, markup, matchAgainst):
975: #print "Matching %s against %s" % (markup, matchAgainst)
976: result = False
977: if matchAgainst is True:
978: result = markup is not None
979: elif callable(matchAgainst):
980: result = matchAgainst(markup)
981: else:
982: #Custom match methods take the tag as an argument, but all
983: #other ways of matching match the tag name as a string.
984: if isinstance(markup, Tag):
985: markup = markup.name
986: if markup and not isinstance(markup, basestring):
987: markup = unicode(markup)
988: #Now we know that chunk is either a string, or None.
989: if hasattr(matchAgainst, 'match'):
990: # It's a regexp object.
991: result = markup and matchAgainst.search(markup)
992: elif hasattr(matchAgainst, '__iter__'): # list-like
993: result = markup in matchAgainst
994: elif hasattr(matchAgainst, 'items'):
995: result = markup.has_key(matchAgainst)
996: elif matchAgainst and isinstance(markup, basestring):
997: if isinstance(markup, unicode):
998: matchAgainst = unicode(matchAgainst)
999: else:
1000: matchAgainst = str(matchAgainst)
1001:
1002: if not result:
1003: result = matchAgainst == markup
1004: return result
1005:
1006: class ResultSet(list):
1007: """A ResultSet is just a list that keeps track of the SoupStrainer
1008: that created it."""
1009: def __init__(self, source):
1010: list.__init__([])
1011: self.source = source
1012:
1013: # Now, some helper functions.
1014:
1015: def buildTagMap(default, *args):
1016: """Turns a list of maps, lists, or scalars into a single map.
1017: Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and
1018: NESTING_RESET_TAGS maps out of lists and partial maps."""
1019: built = {}
1020: for portion in args:
1021: if hasattr(portion, 'items'):
1022: #It's a map. Merge it.
1023: for k,v in portion.items():
1024: built[k] = v
1025: elif hasattr(portion, '__iter__'): # is a list
1026: #It's a list. Map each item to the default.
1027: for k in portion:
1028: built[k] = default
1029: else:
1030: #It's a scalar. Map it to the default.
1031: built[portion] = default
1032: return built
1033:
1034: # Now, the parser classes.
1035:
1036: class BeautifulStoneSoup(Tag, SGMLParser):
1037:
1038: """This class contains the basic parser and search code. It defines
1039: a parser that knows nothing about tag behavior except for the
1040: following:
1041:
1042: You can't close a tag without closing all the tags it encloses.
1043: That is, "<foo><bar></foo>" actually means
1044: "<foo><bar></bar></foo>".
1045:
1046: [Another possible explanation is "<foo><bar /></foo>", but since
1047: this class defines no SELF_CLOSING_TAGS, it will never use that
1048: explanation.]
1049:
1050: This class is useful for parsing XML or made-up markup languages,
1051: or when BeautifulSoup makes an assumption counter to what you were
1052: expecting."""
1053:
1054: SELF_CLOSING_TAGS = {}
1055: NESTABLE_TAGS = {}
1056: RESET_NESTING_TAGS = {}
1057: QUOTE_TAGS = {}
1058: PRESERVE_WHITESPACE_TAGS = []
1059:
1060: MARKUP_MASSAGE = [(re.compile('(<[^<>]*)/>'),
1061: lambda x: x.group(1) + ' />'),
1062: (re.compile('<!\s+([^<>]*)>'),
1063: lambda x: '<!' + x.group(1) + '>')
1064: ]
1065:
1066: ROOT_TAG_NAME = u'[document]'
1067:
1068: HTML_ENTITIES = "html"
1069: XML_ENTITIES = "xml"
1070: XHTML_ENTITIES = "xhtml"
1071: # TODO: This only exists for backwards-compatibility
1072: ALL_ENTITIES = XHTML_ENTITIES
1073:
1074: # Used when determining whether a text node is all whitespace and
1075: # can be replaced with a single space. A text node that contains
1076: # fancy Unicode spaces (usually non-breaking) should be left
1077: # alone.
1078: STRIP_ASCII_SPACES = { 9: None, 10: None, 12: None, 13: None, 32: None, }
1079:
1080: def __init__(self, markup="", parseOnlyThese=None, fromEncoding=None,
1081: markupMassage=True, smartQuotesTo=XML_ENTITIES,
1082: convertEntities=None, selfClosingTags=None, isHTML=False):
1083: """The Soup object is initialized as the 'root tag', and the
1084: provided markup (which can be a string or a file-like object)
1085: is fed into the underlying parser.
1086:
1087: sgmllib will process most bad HTML, and the BeautifulSoup
1088: class has some tricks for dealing with some HTML that kills
1089: sgmllib, but Beautiful Soup can nonetheless choke or lose data
1090: if your data uses self-closing tags or declarations
1091: incorrectly.
1092:
1093: By default, Beautiful Soup uses regexes to sanitize input,
1094: avoiding the vast majority of these problems. If the problems
1095: don't apply to you, pass in False for markupMassage, and
1096: you'll get better performance.
1097:
1098: The default parser massage techniques fix the two most common
1099: instances of invalid HTML that choke sgmllib:
1100:
1101: <br/> (No space between name of closing tag and tag close)
1102: <! --Comment--> (Extraneous whitespace in declaration)
1103:
1104: You can pass in a custom list of (RE object, replace method)
1105: tuples to get Beautiful Soup to scrub your input the way you
1106: want."""
1107:
1108: self.parseOnlyThese = parseOnlyThese
1109: self.fromEncoding = fromEncoding
1110: self.smartQuotesTo = smartQuotesTo
1111: self.convertEntities = convertEntities
1112: # Set the rules for how we'll deal with the entities we
1113: # encounter
1114: if self.convertEntities:
1115: # It doesn't make sense to convert encoded characters to
1116: # entities even while you're converting entities to Unicode.
1117: # Just convert it all to Unicode.
1118: self.smartQuotesTo = None
1119: if convertEntities == self.HTML_ENTITIES:
1120: self.convertXMLEntities = False
1121: self.convertHTMLEntities = True
1122: self.escapeUnrecognizedEntities = True
1123: elif convertEntities == self.XHTML_ENTITIES:
1124: self.convertXMLEntities = True
1125: self.convertHTMLEntities = True
1126: self.escapeUnrecognizedEntities = False
1127: elif convertEntities == self.XML_ENTITIES:
1128: self.convertXMLEntities = True
1129: self.convertHTMLEntities = False
1130: self.escapeUnrecognizedEntities = False
1131: else:
1132: self.convertXMLEntities = False
1133: self.convertHTMLEntities = False
1134: self.escapeUnrecognizedEntities = False
1135:
1136: self.instanceSelfClosingTags = buildTagMap(None, selfClosingTags)
1137: SGMLParser.__init__(self)
1138:
1139: if hasattr(markup, 'read'): # It's a file-type object.
1140: markup = markup.read()
1141: self.markup = markup
1142: self.markupMassage = markupMassage
1143: try:
1144: self._feed(isHTML=isHTML)
1145: except StopParsing:
1146: pass
1147: self.markup = None # The markup can now be GCed
1148:
1149: def convert_charref(self, name):
1150: """This method fixes a bug in Python's SGMLParser."""
1151: try:
1152: n = int(name)
1153: except ValueError:
1154: return
1155: if not 0 <= n <= 127 : # ASCII ends at 127, not 255
1156: return
1157: return self.convert_codepoint(n)
1158:
1159: def _feed(self, inDocumentEncoding=None, isHTML=False):
1160: # Convert the document to Unicode.
1161: markup = self.markup
1162: if isinstance(markup, unicode):
1163: if not hasattr(self, 'originalEncoding'):
1164: self.originalEncoding = None
1165: else:
1166: dammit = UnicodeDammit\
1167: (markup, [self.fromEncoding, inDocumentEncoding],
1168: smartQuotesTo=self.smartQuotesTo, isHTML=isHTML)
1169: markup = dammit.unicode
1170: self.originalEncoding = dammit.originalEncoding
1171: self.declaredHTMLEncoding = dammit.declaredHTMLEncoding
1172: if markup:
1173: if self.markupMassage:
1174: if not hasattr(self.markupMassage, "__iter__"):
1175: self.markupMassage = self.MARKUP_MASSAGE
1176: for fix, m in self.markupMassage:
1177: markup = fix.sub(m, markup)
1178: # TODO: We get rid of markupMassage so that the
1179: # soup object can be deepcopied later on. Some
1180: # Python installations can't copy regexes. If anyone
1181: # was relying on the existence of markupMassage, this
1182: # might cause problems.
1183: del(self.markupMassage)
1184: self.reset()
1185:
1186: SGMLParser.feed(self, markup)
1187: # Close out any unfinished strings and close all the open tags.
1188: self.endData()
1189: while self.currentTag.name != self.ROOT_TAG_NAME:
1190: self.popTag()
1191:
1192: def __getattr__(self, methodName):
1193: """This method routes method call requests to either the SGMLParser
1194: superclass or the Tag superclass, depending on the method name."""
1195: #print "__getattr__ called on %s.%s" % (self.__class__, methodName)
1196:
1197: if methodName.startswith('start_') or methodName.startswith('end_') \
1198: or methodName.startswith('do_'):
1199: return SGMLParser.__getattr__(self, methodName)
1200: elif not methodName.startswith('__'):
1201: return Tag.__getattr__(self, methodName)
1202: else:
1203: raise AttributeError
1204:
1205: def isSelfClosingTag(self, name):
1206: """Returns true iff the given string is the name of a
1207: self-closing tag according to this parser."""
1208: return self.SELF_CLOSING_TAGS.has_key(name) \
1209: or self.instanceSelfClosingTags.has_key(name)
1210:
1211: def reset(self):
1212: Tag.__init__(self, self, self.ROOT_TAG_NAME)
1213: self.hidden = 1
1214: SGMLParser.reset(self)
1215: self.currentData = []
1216: self.currentTag = None
1217: self.tagStack = []
1218: self.quoteStack = []
1219: self.pushTag(self)
1220:
1221: def popTag(self):
1222: tag = self.tagStack.pop()
1223:
1224: #print "Pop", tag.name
1225: if self.tagStack:
1226: self.currentTag = self.tagStack[-1]
1227: return self.currentTag
1228:
1229: def pushTag(self, tag):
1230: #print "Push", tag.name
1231: if self.currentTag:
1232: self.currentTag.contents.append(tag)
1233: self.tagStack.append(tag)
1234: self.currentTag = self.tagStack[-1]
1235:
1236: def endData(self, containerClass=NavigableString):
1237: if self.currentData:
1238: currentData = u''.join(self.currentData)
1239: if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and
1240: not set([tag.name for tag in self.tagStack]).intersection(
1241: self.PRESERVE_WHITESPACE_TAGS)):
1242: if '\n' in currentData:
1243: currentData = '\n'
1244: else:
1245: currentData = ' '
1246: self.currentData = []
1247: if self.parseOnlyThese and len(self.tagStack) <= 1 and \
1248: (not self.parseOnlyThese.text or \
1249: not self.parseOnlyThese.search(currentData)):
1250: return
1251: o = containerClass(currentData)
1252: o.setup(self.currentTag, self.previous)
1253: if self.previous:
1254: self.previous.next = o
1255: self.previous = o
1256: self.currentTag.contents.append(o)
1257:
1258:
1259: def _popToTag(self, name, inclusivePop=True):
1260: """Pops the tag stack up to and including the most recent
1261: instance of the given tag. If inclusivePop is false, pops the tag
1262: stack up to but *not* including the most recent instqance of
1263: the given tag."""
1264: #print "Popping to %s" % name
1265: if name == self.ROOT_TAG_NAME:
1266: return
1267:
1268: numPops = 0
1269: mostRecentTag = None
1270: for i in range(len(self.tagStack)-1, 0, -1):
1271: if name == self.tagStack[i].name:
1272: numPops = len(self.tagStack)-i
1273: break
1274: if not inclusivePop:
1275: numPops = numPops - 1
1276:
1277: for i in range(0, numPops):
1278: mostRecentTag = self.popTag()
1279: return mostRecentTag
1280:
1281: def _smartPop(self, name):
1282:
1283: """We need to pop up to the previous tag of this type, unless
1284: one of this tag's nesting reset triggers comes between this
1285: tag and the previous tag of this type, OR unless this tag is a
1286: generic nesting trigger and another generic nesting trigger
1287: comes between this tag and the previous tag of this type.
1288:
1289: Examples:
1290: <p>Foo<b>Bar *<p>* should pop to 'p', not 'b'.
1291: <p>Foo<table>Bar *<p>* should pop to 'table', not 'p'.
1292: <p>Foo<table><tr>Bar *<p>* should pop to 'tr', not 'p'.
1293:
1294: <li><ul><li> *<li>* should pop to 'ul', not the first 'li'.
1295: <tr><table><tr> *<tr>* should pop to 'table', not the first 'tr'
1296: <td><tr><td> *<td>* should pop to 'tr', not the first 'td'
1297: """
1298:
1299: nestingResetTriggers = self.NESTABLE_TAGS.get(name)
1300: isNestable = nestingResetTriggers != None
1301: isResetNesting = self.RESET_NESTING_TAGS.has_key(name)
1302: popTo = None
1303: inclusive = True
1304: for i in range(len(self.tagStack)-1, 0, -1):
1305: p = self.tagStack[i]
1306: if (not p or p.name == name) and not isNestable:
1307: #Non-nestable tags get popped to the top or to their
1308: #last occurance.
1309: popTo = name
1310: break
1311: if (nestingResetTriggers is not None
1312: and p.name in nestingResetTriggers) \
1313: or (nestingResetTriggers is None and isResetNesting
1314: and self.RESET_NESTING_TAGS.has_key(p.name)):
1315:
1316: #If we encounter one of the nesting reset triggers
1317: #peculiar to this tag, or we encounter another tag
1318: #that causes nesting to reset, pop up to but not
1319: #including that tag.
1320: popTo = p.name
1321: inclusive = False
1322: break
1323: p = p.parent
1324: if popTo:
1325: self._popToTag(popTo, inclusive)
1326:
1327: def unknown_starttag(self, name, attrs, selfClosing=0):
1328: #print "Start tag %s: %s" % (name, attrs)
1329: if self.quoteStack:
1330: #This is not a real tag.
1331: #print "<%s> is not real!" % name
1332: attrs = ''.join([' %s="%s"' % (x, y) for x, y in attrs])
1333: self.handle_data('<%s%s>' % (name, attrs))
1334: return
1335: self.endData()
1336:
1337: if not self.isSelfClosingTag(name) and not selfClosing:
1338: self._smartPop(name)
1339:
1340: if self.parseOnlyThese and len(self.tagStack) <= 1 \
1341: and (self.parseOnlyThese.text or not self.parseOnlyThese.searchTag(name, attrs)):
1342: return
1343:
1344: tag = Tag(self, name, attrs, self.currentTag, self.previous)
1345: if self.previous:
1346: self.previous.next = tag
1347: self.previous = tag
1348: self.pushTag(tag)
1349: if selfClosing or self.isSelfClosingTag(name):
1350: self.popTag()
1351: if name in self.QUOTE_TAGS:
1352: #print "Beginning quote (%s)" % name
1353: self.quoteStack.append(name)
1354: self.literal = 1
1355: return tag
1356:
1357: def unknown_endtag(self, name):
1358: #print "End tag %s" % name
1359: if self.quoteStack and self.quoteStack[-1] != name:
1360: #This is not a real end tag.
1361: #print "</%s> is not real!" % name
1362: self.handle_data('</%s>' % name)
1363: return
1364: self.endData()
1365: self._popToTag(name)
1366: if self.quoteStack and self.quoteStack[-1] == name:
1367: self.quoteStack.pop()
1368: self.literal = (len(self.quoteStack) > 0)
1369:
1370: def handle_data(self, data):
1371: self.currentData.append(data)
1372:
1373: def _toStringSubclass(self, text, subclass):
1374: """Adds a certain piece of text to the tree as a NavigableString
1375: subclass."""
1376: self.endData()
1377: self.handle_data(text)
1378: self.endData(subclass)
1379:
1380: def handle_pi(self, text):
1381: """Handle a processing instruction as a ProcessingInstruction
1382: object, possibly one with a %SOUP-ENCODING% slot into which an
1383: encoding will be plugged later."""
1384: if text[:3] == "xml":
1385: text = u"xml version='1.0' encoding='%SOUP-ENCODING%'"
1386: self._toStringSubclass(text, ProcessingInstruction)
1387:
1388: def handle_comment(self, text):
1389: "Handle comments as Comment objects."
1390: self._toStringSubclass(text, Comment)
1391:
1392: def handle_charref(self, ref):
1393: "Handle character references as data."
1394: if self.convertEntities:
1395: data = unichr(int(ref))
1396: else:
1397: data = '&#%s;' % ref
1398: self.handle_data(data)
1399:
1400: def handle_entityref(self, ref):
1401: """Handle entity references as data, possibly converting known
1402: HTML and/or XML entity references to the corresponding Unicode
1403: characters."""
1404: data = None
1405: if self.convertHTMLEntities:
1406: try:
1407: data = unichr(name2codepoint[ref])
1408: except KeyError:
1409: pass
1410:
1411: if not data and self.convertXMLEntities:
1412: data = self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref)
1413:
1414: if not data and self.convertHTMLEntities and \
1415: not self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref):
1416: # TODO: We've got a problem here. We're told this is
1417: # an entity reference, but it's not an XML entity
1418: # reference or an HTML entity reference. Nonetheless,
1419: # the logical thing to do is to pass it through as an
1420: # unrecognized entity reference.
1421: #
1422: # Except: when the input is "&carol;" this function
1423: # will be called with input "carol". When the input is
1424: # "AT&T", this function will be called with input
1425: # "T". We have no way of knowing whether a semicolon
1426: # was present originally, so we don't know whether
1427: # this is an unknown entity or just a misplaced
1428: # ampersand.
1429: #
1430: # The more common case is a misplaced ampersand, so I
1431: # escape the ampersand and omit the trailing semicolon.
1432: data = "&%s" % ref
1433: if not data:
1434: # This case is different from the one above, because we
1435: # haven't already gone through a supposedly comprehensive
1436: # mapping of entities to Unicode characters. We might not
1437: # have gone through any mapping at all. So the chances are
1438: # very high that this is a real entity, and not a
1439: # misplaced ampersand.
1440: data = "&%s;" % ref
1441: self.handle_data(data)
1442:
1443: def handle_decl(self, data):
1444: "Handle DOCTYPEs and the like as Declaration objects."
1445: self._toStringSubclass(data, Declaration)
1446:
1447: def parse_declaration(self, i):
1448: """Treat a bogus SGML declaration as raw data. Treat a CDATA
1449: declaration as a CData object."""
1450: j = None
1451: if self.rawdata[i:i+9] == '<![CDATA[':
1452: k = self.rawdata.find(']]>', i)
1453: if k == -1:
1454: k = len(self.rawdata)
1455: data = self.rawdata[i+9:k]
1456: j = k+3
1457: self._toStringSubclass(data, CData)
1458: else:
1459: try:
1460: j = SGMLParser.parse_declaration(self, i)
1461: except SGMLParseError:
1462: toHandle = self.rawdata[i:]
1463: self.handle_data(toHandle)
1464: j = i + len(toHandle)
1465: return j
1466:
1467: class BeautifulSoup(BeautifulStoneSoup):
1468:
1469: """This parser knows the following facts about HTML:
1470:
1471: * Some tags have no closing tag and should be interpreted as being
1472: closed as soon as they are encountered.
1473:
1474: * The text inside some tags (ie. 'script') may contain tags which
1475: are not really part of the document and which should be parsed
1476: as text, not tags. If you want to parse the text as tags, you can
1477: always fetch it and parse it explicitly.
1478:
1479: * Tag nesting rules:
1480:
1481: Most tags can't be nested at all. For instance, the occurance of
1482: a <p> tag should implicitly close the previous <p> tag.
1483:
1484: <p>Para1<p>Para2
1485: should be transformed into:
1486: <p>Para1</p><p>Para2
1487:
1488: Some tags can be nested arbitrarily. For instance, the occurance
1489: of a <blockquote> tag should _not_ implicitly close the previous
1490: <blockquote> tag.
1491:
1492: Alice said: <blockquote>Bob said: <blockquote>Blah
1493: should NOT be transformed into:
1494: Alice said: <blockquote>Bob said: </blockquote><blockquote>Blah
1495:
1496: Some tags can be nested, but the nesting is reset by the
1497: interposition of other tags. For instance, a <tr> tag should
1498: implicitly close the previous <tr> tag within the same <table>,
1499: but not close a <tr> tag in another table.
1500:
1501: <table><tr>Blah<tr>Blah
1502: should be transformed into:
1503: <table><tr>Blah</tr><tr>Blah
1504: but,
1505: <tr>Blah<table><tr>Blah
1506: should NOT be transformed into
1507: <tr>Blah<table></tr><tr>Blah
1508:
1509: Differing assumptions about tag nesting rules are a major source
1510: of problems with the BeautifulSoup class. If BeautifulSoup is not
1511: treating as nestable a tag your page author treats as nestable,
1512: try ICantBelieveItsBeautifulSoup, MinimalSoup, or
1513: BeautifulStoneSoup before writing your own subclass."""
1514:
1515: def __init__(self, *args, **kwargs):
1516: if not kwargs.has_key('smartQuotesTo'):
1517: kwargs['smartQuotesTo'] = self.HTML_ENTITIES
1518: kwargs['isHTML'] = True
1519: BeautifulStoneSoup.__init__(self, *args, **kwargs)
1520:
1521: SELF_CLOSING_TAGS = buildTagMap(None,
1522: ('br' , 'hr', 'input', 'img', 'meta',
1523: 'spacer', 'link', 'frame', 'base', 'col'))
1524:
1525: PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea'])
1526:
1527: QUOTE_TAGS = {'script' : None, 'textarea' : None}
1528:
1529: #According to the HTML standard, each of these inline tags can
1530: #contain another tag of the same type. Furthermore, it's common
1531: #to actually use these tags this way.
1532: NESTABLE_INLINE_TAGS = ('span', 'font', 'q', 'object', 'bdo', 'sub', 'sup',
1533: 'center')
1534:
1535: #According to the HTML standard, these block tags can contain
1536: #another tag of the same type. Furthermore, it's common
1537: #to actually use these tags this way.
1538: NESTABLE_BLOCK_TAGS = ('blockquote', 'div', 'fieldset', 'ins', 'del')
1539:
1540: #Lists can contain other lists, but there are restrictions.
1541: NESTABLE_LIST_TAGS = { 'ol' : [],
1542: 'ul' : [],
1543: 'li' : ['ul', 'ol'],
1544: 'dl' : [],
1545: 'dd' : ['dl'],
1546: 'dt' : ['dl'] }
1547:
1548: #Tables can contain other tables, but there are restrictions.
1549: NESTABLE_TABLE_TAGS = {'table' : [],
1550: 'tr' : ['table', 'tbody', 'tfoot', 'thead'],
1551: 'td' : ['tr'],
1552: 'th' : ['tr'],
1553: 'thead' : ['table'],
1554: 'tbody' : ['table'],
1555: 'tfoot' : ['table'],
1556: }
1557:
1558: NON_NESTABLE_BLOCK_TAGS = ('address', 'form', 'p', 'pre')
1559:
1560: #If one of these tags is encountered, all tags up to the next tag of
1561: #this type are popped.
1562: RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript',
1563: NON_NESTABLE_BLOCK_TAGS,
1564: NESTABLE_LIST_TAGS,
1565: NESTABLE_TABLE_TAGS)
1566:
1567: NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS,
1568: NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS)
1569:
1570: # Used to detect the charset in a META tag; see start_meta
1571: CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)", re.M)
1572:
1573: def start_meta(self, attrs):
1574: """Beautiful Soup can detect a charset included in a META tag,
1575: try to convert the document to that charset, and re-parse the
1576: document from the beginning."""
1577: httpEquiv = None
1578: contentType = None
1579: contentTypeIndex = None
1580: tagNeedsEncodingSubstitution = False
1581:
1582: for i in range(0, len(attrs)):
1583: key, value = attrs[i]
1584: key = key.lower()
1585: if key == 'http-equiv':
1586: httpEquiv = value
1587: elif key == 'content':
1588: contentType = value
1589: contentTypeIndex = i
1590:
1591: if httpEquiv and contentType: # It's an interesting meta tag.
1592: match = self.CHARSET_RE.search(contentType)
1593: if match:
1594: if (self.declaredHTMLEncoding is not None or
1595: self.originalEncoding == self.fromEncoding):
1596: # An HTML encoding was sniffed while converting
1597: # the document to Unicode, or an HTML encoding was
1598: # sniffed during a previous pass through the
1599: # document, or an encoding was specified
1600: # explicitly and it worked. Rewrite the meta tag.
1601: def rewrite(match):
1602: return match.group(1) + "%SOUP-ENCODING%"
1603: newAttr = self.CHARSET_RE.sub(rewrite, contentType)
1604: attrs[contentTypeIndex] = (attrs[contentTypeIndex][0],
1605: newAttr)
1606: tagNeedsEncodingSubstitution = True
1607: else:
1608: # This is our first pass through the document.
1609: # Go through it again with the encoding information.
1610: newCharset = match.group(3)
1611: if newCharset and newCharset != self.originalEncoding:
1612: self.declaredHTMLEncoding = newCharset
1613: self._feed(self.declaredHTMLEncoding)
1614: raise StopParsing
1615: pass
1616: tag = self.unknown_starttag("meta", attrs)
1617: if tag and tagNeedsEncodingSubstitution:
1618: tag.containsSubstitutions = True
1619:
1620: class StopParsing(Exception):
1621: pass
1622:
1623: class ICantBelieveItsBeautifulSoup(BeautifulSoup):
1624:
1625: """The BeautifulSoup class is oriented towards skipping over
1626: common HTML errors like unclosed tags. However, sometimes it makes
1627: errors of its own. For instance, consider this fragment:
1628:
1629: <b>Foo<b>Bar</b></b>
1630:
1631: This is perfectly valid (if bizarre) HTML. However, the
1632: BeautifulSoup class will implicitly close the first b tag when it
1633: encounters the second 'b'. It will think the author wrote
1634: "<b>Foo<b>Bar", and didn't close the first 'b' tag, because
1635: there's no real-world reason to bold something that's already
1636: bold. When it encounters '</b></b>' it will close two more 'b'
1637: tags, for a grand total of three tags closed instead of two. This
1638: can throw off the rest of your document structure. The same is
1639: true of a number of other tags, listed below.
1640:
1641: It's much more common for someone to forget to close a 'b' tag
1642: than to actually use nested 'b' tags, and the BeautifulSoup class
1643: handles the common case. This class handles the not-co-common
1644: case: where you can't believe someone wrote what they did, but
1645: it's valid HTML and BeautifulSoup screwed up by assuming it
1646: wouldn't be."""
1647:
1648: I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \
1649: ('em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong',
1650: 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b',
1651: 'big')
1652:
1653: I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ('noscript',)
1654:
1655: NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS,
1656: I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS,
1657: I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS)
1658:
1659: class MinimalSoup(BeautifulSoup):
1660: """The MinimalSoup class is for parsing HTML that contains
1661: pathologically bad markup. It makes no assumptions about tag
1662: nesting, but it does know which tags are self-closing, that
1663: <script> tags contain Javascript and should not be parsed, that
1664: META tags may contain encoding information, and so on.
1665:
1666: This also makes it better for subclassing than BeautifulStoneSoup
1667: or BeautifulSoup."""
1668:
1669: RESET_NESTING_TAGS = buildTagMap('noscript')
1670: NESTABLE_TAGS = {}
1671:
1672: class BeautifulSOAP(BeautifulStoneSoup):
1673: """This class will push a tag with only a single string child into
1674: the tag's parent as an attribute. The attribute's name is the tag
1675: name, and the value is the string child. An example should give
1676: the flavor of the change:
1677:
1678: <foo><bar>baz</bar></foo>
1679: =>
1680: <foo bar="baz"><bar>baz</bar></foo>
1681:
1682: You can then access fooTag['bar'] instead of fooTag.barTag.string.
1683:
1684: This is, of course, useful for scraping structures that tend to
1685: use subelements instead of attributes, such as SOAP messages. Note
1686: that it modifies its input, so don't print the modified version
1687: out.
1688:
1689: I'm not sure how many people really want to use this class; let me
1690: know if you do. Mainly I like the name."""
1691:
1692: def popTag(self):
1693: if len(self.tagStack) > 1:
1694: tag = self.tagStack[-1]
1695: parent = self.tagStack[-2]
1696: parent._getAttrMap()
1697: if (isinstance(tag, Tag) and len(tag.contents) == 1 and
1698: isinstance(tag.contents[0], NavigableString) and
1699: not parent.attrMap.has_key(tag.name)):
1700: parent[tag.name] = tag.contents[0]
1701: BeautifulStoneSoup.popTag(self)
1702:
1703: #Enterprise class names! It has come to our attention that some people
1704: #think the names of the Beautiful Soup parser classes are too silly
1705: #and "unprofessional" for use in enterprise screen-scraping. We feel
1706: #your pain! For such-minded folk, the Beautiful Soup Consortium And
1707: #All-Night Kosher Bakery recommends renaming this file to
1708: #"RobustParser.py" (or, in cases of extreme enterprisiness,
1709: #"RobustParserBeanInterface.class") and using the following
1710: #enterprise-friendly class aliases:
1711: class RobustXMLParser(BeautifulStoneSoup):
1712: pass
1713: class RobustHTMLParser(BeautifulSoup):
1714: pass
1715: class RobustWackAssHTMLParser(ICantBelieveItsBeautifulSoup):
1716: pass
1717: class RobustInsanelyWackAssHTMLParser(MinimalSoup):
1718: pass
1719: class SimplifyingSOAPParser(BeautifulSOAP):
1720: pass
1721:
1722: ######################################################
1723: #
1724: # Bonus library: Unicode, Dammit
1725: #
1726: # This class forces XML data into a standard format (usually to UTF-8
1727: # or Unicode). It is heavily based on code from Mark Pilgrim's
1728: # Universal Feed Parser. It does not rewrite the XML or HTML to
1729: # reflect a new encoding: that happens in BeautifulStoneSoup.handle_pi
1730: # (XML) and BeautifulSoup.start_meta (HTML).
1731:
1732: # Autodetects character encodings.
1733: # Download from http://chardet.feedparser.org/
1734: try:
1735: import chardet
1736: # import chardet.constants
1737: # chardet.constants._debug = 1
1738: except ImportError:
1739: chardet = None
1740:
1741: # cjkcodecs and iconv_codec make Python know about more character encodings.
1742: # Both are available from http://cjkpython.i18n.org/
1743: # They're built in if you use Python 2.4.
1744: try:
1745: import cjkcodecs.aliases
1746: except ImportError:
1747: pass
1748: try:
1749: import iconv_codec
1750: except ImportError:
1751: pass
1752:
1753: class UnicodeDammit:
1754: """A class for detecting the encoding of a *ML document and
1755: converting it to a Unicode string. If the source encoding is
1756: windows-1252, can replace MS smart quotes with their HTML or XML
1757: equivalents."""
1758:
1759: # This dictionary maps commonly seen values for "charset" in HTML
1760: # meta tags to the corresponding Python codec names. It only covers
1761: # values that aren't in Python's aliases and can't be determined
1762: # by the heuristics in find_codec.
1763: CHARSET_ALIASES = { "macintosh" : "mac-roman",
1764: "x-sjis" : "shift-jis" }
1765:
1766: def __init__(self, markup, overrideEncodings=[],
1767: smartQuotesTo='xml', isHTML=False):
1768: self.declaredHTMLEncoding = None
1769: self.markup, documentEncoding, sniffedEncoding = \
1770: self._detectEncoding(markup, isHTML)
1771: self.smartQuotesTo = smartQuotesTo
1772: self.triedEncodings = []
1773: if markup == '' or isinstance(markup, unicode):
1774: self.originalEncoding = None
1775: self.unicode = unicode(markup)
1776: return
1777:
1778: u = None
1779: for proposedEncoding in overrideEncodings:
1780: u = self._convertFrom(proposedEncoding)
1781: if u: break
1782: if not u:
1783: for proposedEncoding in (documentEncoding, sniffedEncoding):
1784: u = self._convertFrom(proposedEncoding)
1785: if u: break
1786:
1787: # If no luck and we have auto-detection library, try that:
1788: if not u and chardet and not isinstance(self.markup, unicode):
1789: u = self._convertFrom(chardet.detect(self.markup)['encoding'])
1790:
1791: # As a last resort, try utf-8 and windows-1252:
1792: if not u:
1793: for proposed_encoding in ("utf-8", "windows-1252"):
1794: u = self._convertFrom(proposed_encoding)
1795: if u: break
1796:
1797: self.unicode = u
1798: if not u: self.originalEncoding = None
1799:
1800: def _subMSChar(self, orig):
1801: """Changes a MS smart quote character to an XML or HTML
1802: entity."""
1803: sub = self.MS_CHARS.get(orig)
1804: if isinstance(sub, tuple):
1805: if self.smartQuotesTo == 'xml':
1806: sub = '&#x%s;' % sub[1]
1807: else:
1808: sub = '&%s;' % sub[0]
1809: return sub
1810:
1811: def _convertFrom(self, proposed):
1812: proposed = self.find_codec(proposed)
1813: if not proposed or proposed in self.triedEncodings:
1814: return None
1815: self.triedEncodings.append(proposed)
1816: markup = self.markup
1817:
1818: # Convert smart quotes to HTML if coming from an encoding
1819: # that might have them.
1820: if self.smartQuotesTo and proposed.lower() in("windows-1252",
1821: "iso-8859-1",
1822: "iso-8859-2"):
1823: markup = re.compile("([\x80-\x9f])").sub \
1824: (lambda(x): self._subMSChar(x.group(1)),
1825: markup)
1826:
1827: try:
1828: # print "Trying to convert document to %s" % proposed
1829: u = self._toUnicode(markup, proposed)
1830: self.markup = u
1831: self.originalEncoding = proposed
1832: except Exception, e:
1833: # print "That didn't work!"
1834: # print e
1835: return None
1836: #print "Correct encoding: %s" % proposed
1837: return self.markup
1838:
1839: def _toUnicode(self, data, encoding):
1840: '''Given a string and its encoding, decodes the string into Unicode.
1841: %encoding is a string recognized by encodings.aliases'''
1842:
1843: # strip Byte Order Mark (if present)
1844: if (len(data) >= 4) and (data[:2] == '\xfe\xff') \
1845: and (data[2:4] != '\x00\x00'):
1846: encoding = 'utf-16be'
1847: data = data[2:]
1848: elif (len(data) >= 4) and (data[:2] == '\xff\xfe') \
1849: and (data[2:4] != '\x00\x00'):
1850: encoding = 'utf-16le'
1851: data = data[2:]
1852: elif data[:3] == '\xef\xbb\xbf':
1853: encoding = 'utf-8'
1854: data = data[3:]
1855: elif data[:4] == '\x00\x00\xfe\xff':
1856: encoding = 'utf-32be'
1857: data = data[4:]
1858: elif data[:4] == '\xff\xfe\x00\x00':
1859: encoding = 'utf-32le'
1860: data = data[4:]
1861: newdata = unicode(data, encoding)
1862: return newdata
1863:
1864: def _detectEncoding(self, xml_data, isHTML=False):
1865: """Given a document, tries to detect its XML encoding."""
1866: xml_encoding = sniffed_xml_encoding = None
1867: try:
1868: if xml_data[:4] == '\x4c\x6f\xa7\x94':
1869: # EBCDIC
1870: xml_data = self._ebcdic_to_ascii(xml_data)
1871: elif xml_data[:4] == '\x00\x3c\x00\x3f':
1872: # UTF-16BE
1873: sniffed_xml_encoding = 'utf-16be'
1874: xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
1875: elif (len(xml_data) >= 4) and (xml_data[:2] == '\xfe\xff') \
1876: and (xml_data[2:4] != '\x00\x00'):
1877: # UTF-16BE with BOM
1878: sniffed_xml_encoding = 'utf-16be'
1879: xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
1880: elif xml_data[:4] == '\x3c\x00\x3f\x00':
1881: # UTF-16LE
1882: sniffed_xml_encoding = 'utf-16le'
1883: xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
1884: elif (len(xml_data) >= 4) and (xml_data[:2] == '\xff\xfe') and \
1885: (xml_data[2:4] != '\x00\x00'):
1886: # UTF-16LE with BOM
1887: sniffed_xml_encoding = 'utf-16le'
1888: xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
1889: elif xml_data[:4] == '\x00\x00\x00\x3c':
1890: # UTF-32BE
1891: sniffed_xml_encoding = 'utf-32be'
1892: xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
1893: elif xml_data[:4] == '\x3c\x00\x00\x00':
1894: # UTF-32LE
1895: sniffed_xml_encoding = 'utf-32le'
1896: xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
1897: elif xml_data[:4] == '\x00\x00\xfe\xff':
1898: # UTF-32BE with BOM
1899: sniffed_xml_encoding = 'utf-32be'
1900: xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
1901: elif xml_data[:4] == '\xff\xfe\x00\x00':
1902: # UTF-32LE with BOM
1903: sniffed_xml_encoding = 'utf-32le'
1904: xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
1905: elif xml_data[:3] == '\xef\xbb\xbf':
1906: # UTF-8 with BOM
1907: sniffed_xml_encoding = 'utf-8'
1908: xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
1909: else:
1910: sniffed_xml_encoding = 'ascii'
1911: pass
1912: except:
1913: xml_encoding_match = None
1914: xml_encoding_match = re.compile(
1915: '^<\?.*encoding=[\'"](.*?)[\'"].*\?>').match(xml_data)
1916: if not xml_encoding_match and isHTML:
1917: regexp = re.compile('<\s*meta[^>]+charset=([^>]*?)[;\'">]', re.I)
1918: xml_encoding_match = regexp.search(xml_data)
1919: if xml_encoding_match is not None:
1920: xml_encoding = xml_encoding_match.groups()[0].lower()
1921: if isHTML:
1922: self.declaredHTMLEncoding = xml_encoding
1923: if sniffed_xml_encoding and \
1924: (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
1925: 'iso-10646-ucs-4', 'ucs-4', 'csucs4',
1926: 'utf-16', 'utf-32', 'utf_16', 'utf_32',
1927: 'utf16', 'u16')):
1928: xml_encoding = sniffed_xml_encoding
1929: return xml_data, xml_encoding, sniffed_xml_encoding
1930:
1931:
1932: def find_codec(self, charset):
1933: return self._codec(self.CHARSET_ALIASES.get(charset, charset)) \
1934: or (charset and self._codec(charset.replace("-", ""))) \
1935: or (charset and self._codec(charset.replace("-", "_"))) \
1936: or charset
1937:
1938: def _codec(self, charset):
1939: if not charset: return charset
1940: codec = None
1941: try:
1942: codecs.lookup(charset)
1943: codec = charset
1944: except (LookupError, ValueError):
1945: pass
1946: return codec
1947:
1948: EBCDIC_TO_ASCII_MAP = None
1949: def _ebcdic_to_ascii(self, s):
1950: c = self.__class__
1951: if not c.EBCDIC_TO_ASCII_MAP:
1952: emap = (0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
1953: 16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
1954: 128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
1955: 144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
1956: 32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
1957: 38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
1958: 45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
1959: 186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
1960: 195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,
1961: 201,202,106,107,108,109,110,111,112,113,114,203,204,205,
1962: 206,207,208,209,126,115,116,117,118,119,120,121,122,210,
1963: 211,212,213,214,215,216,217,218,219,220,221,222,223,224,
1964: 225,226,227,228,229,230,231,123,65,66,67,68,69,70,71,72,
1965: 73,232,233,234,235,236,237,125,74,75,76,77,78,79,80,81,
1966: 82,238,239,240,241,242,243,92,159,83,84,85,86,87,88,89,
1967: 90,244,245,246,247,248,249,48,49,50,51,52,53,54,55,56,57,
1968: 250,251,252,253,254,255)
1969: import string
1970: c.EBCDIC_TO_ASCII_MAP = string.maketrans( \
1971: ''.join(map(chr, range(256))), ''.join(map(chr, emap)))
1972: return s.translate(c.EBCDIC_TO_ASCII_MAP)
1973:
1974: MS_CHARS = { '\x80' : ('euro', '20AC'),
1975: '\x81' : ' ',
1976: '\x82' : ('sbquo', '201A'),
1977: '\x83' : ('fnof', '192'),
1978: '\x84' : ('bdquo', '201E'),
1979: '\x85' : ('hellip', '2026'),
1980: '\x86' : ('dagger', '2020'),
1981: '\x87' : ('Dagger', '2021'),
1982: '\x88' : ('circ', '2C6'),
1983: '\x89' : ('permil', '2030'),
1984: '\x8A' : ('Scaron', '160'),
1985: '\x8B' : ('lsaquo', '2039'),
1986: '\x8C' : ('OElig', '152'),
1987: '\x8D' : '?',
1988: '\x8E' : ('#x17D', '17D'),
1989: '\x8F' : '?',
1990: '\x90' : '?',
1991: '\x91' : ('lsquo', '2018'),
1992: '\x92' : ('rsquo', '2019'),
1993: '\x93' : ('ldquo', '201C'),
1994: '\x94' : ('rdquo', '201D'),
1995: '\x95' : ('bull', '2022'),
1996: '\x96' : ('ndash', '2013'),
1997: '\x97' : ('mdash', '2014'),
1998: '\x98' : ('tilde', '2DC'),
1999: '\x99' : ('trade', '2122'),
2000: '\x9a' : ('scaron', '161'),
2001: '\x9b' : ('rsaquo', '203A'),
2002: '\x9c' : ('oelig', '153'),
2003: '\x9d' : '?',
2004: '\x9e' : ('#x17E', '17E'),
2005: '\x9f' : ('Yuml', ''),}
2006:
2007: #######################################################################
2008:
2009:
2010: #By default, act as an HTML pretty-printer.
2011: if __name__ == '__main__':
2012: import sys
2013: soup = BeautifulSoup(sys.stdin)
2014: print soup.prettify()
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/cvs/cvsweb/icons/back.gif){kind=icon}
Return to BeautifulSoup.py CVS log ![[TXT]](/cvs/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvs/cvsweb/icons/dir.gif){kind=icon}
Up to [Repository] / MPIWGWeb