Revision: 4092
Author: cosoleto
Date: 2007-08-23 08:30:49 +0000 (Thu, 23 Aug 2007)
Log Message:
-----------
economize_query() to skip sequence of numbers or wide comma separated lists.
Modified Paths:
--------------
trunk/pywikipedia/config.py
trunk/pywikipedia/copyright.py
Modified: trunk/pywikipedia/config.py
===================================================================
--- trunk/pywikipedia/config.py 2007-08-22 21:52:43 UTC (rev 4091)
+++ trunk/pywikipedia/config.py 2007-08-23 08:30:49 UTC (rev 4092)
@@ -344,6 +344,14 @@
# Append length of URL to script result
copyright_show_length = True
+# By default the script try to identify and skip text that contents a wide
+# comma separated list or only numbers. But sometimes that might be the
+# only part unmodified of a slightly edited and not otherwise reported
+# copyright violation. You can disable this feature to try to increase
+# accuracy.
+
+copyright_economize_query = True
+
############## FURTHER SETTINGS ##############
# The bot can make some additional changes to each page it edits, e.g. fix
Modified: trunk/pywikipedia/copyright.py
===================================================================
--- trunk/pywikipedia/copyright.py 2007-08-22 21:52:43 UTC (rev 4091)
+++ trunk/pywikipedia/copyright.py 2007-08-23 08:30:49 UTC (rev 4092)
@@ -75,10 +75,15 @@
__version__='$Id$'
-# Try to skip quoted text
+# Try to skip quoted text.
exclude_quote = True
-# No checks if the page is a disambiguation page
+# If ratio between query length and number of commas is greater or equal
+# to 'comma_ratio' then the script identify a comma separated list and
+# don't send data to search engine.
+comma_ratio = 5
+
+# No checks if the page is a disambiguation page.
skip_disambig = True
appdir = "copyright/"
@@ -323,6 +328,27 @@
f.close()
#
+# Ignore text that contents comma separated list, only numbers,
+# punctuation...
+
+def economize_query(text)
+ # Comma separated list
+ if text.count(', ') > 4:
+ l = len(text)
+ c = text.count(', ')
+ r = 100 * c / l
+
+ if r >= comma_ratio
+ return True
+
+ # write_log("%d/%d/%d: %s\n" % (l,c,r,text), "copyright/skip"
+ str(r) + ".txt")
+
+ # Numbers
+ if re.search('[^0-9\'*/,. +?:;-]{5}', text):
+ return False
+ return True
+
+#
# Set regex used in cleanwikicode() to remove [[Image:]] tags
# and regex used in check_in_source() to reject pages with
# 'Wikipedia'.
@@ -442,6 +468,11 @@
line = cleanwikicode(line)
for search_words in mysplit(line, 31, " "):
if len(search_words) > 120:
+ if config.copyright_economize_query:
+ if economize_query(search_words):
+ wikipedia.output('SKIP TEXT: ' + search_words)
+ consecutive = False
+ continue
n_query += 1
#wikipedia.output(search_words)
if config.copyright_max_query_for_page and n_query >
config.copyright_max_query_for_page: