Revision: 8339
Author: xqt
Date: 2010-06-26 13:48:42 +0000 (Sat, 26 Jun 2010)
Log Message:
-----------
charset detection from http header (patch bug#3021568)
Modified Paths:
--------------
trunk/pywikipedia/reflinks.py
Modified: trunk/pywikipedia/reflinks.py
===================================================================
--- trunk/pywikipedia/reflinks.py 2010-06-26 13:20:38 UTC (rev 8338)
+++ trunk/pywikipedia/reflinks.py 2010-06-26 13:48:42 UTC (rev 8339)
@@ -580,25 +580,32 @@
meta_content = self.META_CONTENT.search(linkedpagetext)
enc = []
+ # use charset from http header
+ s = self.CHARSET.search(contentType)
if meta_content:
tag = meta_content.group()
# Prefer the contentType from the HTTP header :
if not contentType:
contentType = tag
- s = self.CHARSET.search(tag)
- if s:
- tmp = s.group('enc').strip("\"'
").lower()
- naked = re.sub('[ _\-]', '', tmp)
- # Convert to python correct encoding names
- if naked == "gb2312":
- enc.append("gbk")
- elif naked == "shiftjis":
- enc.append("shift jis 2004")
- enc.append("cp932")
- elif naked == "xeucjp":
- enc.append("euc-jp")
- else:
- enc.append(tmp)
+ if not s:
+ # use charset from html
+ s = self.CHARSET.search(tag)
+ if s:
+ tmp = s.group('enc').strip("\"'
").lower()
+ naked = re.sub('[ _\-]', '', tmp)
+ # Convert to python correct encoding names
+ if naked == "gb2312":
+ enc.append("gbk")
+ elif naked == "shiftjis":
+ enc.append("shift jis 2004")
+ enc.append("cp932")
+ elif naked == "xeucjp":
+ enc.append("euc-jp")
+ else:
+ enc.append(tmp)
+ else:
+ wikipedia.output(u'No charset found for %s' % ref.link)
+ #continue # do not process pages without charset
if not contentType:
wikipedia.output(u'No content-type found for %s' % ref.link)
continue