Python code to index a site…

#!C:/Python25/python.exe -u
import MySQLdb

print “Content-type: text/html”
db=MySQLdb.connect(host=”localhost”,user=”xxxxx”,passwd=”xxxxxx”,db=”xxxxx”)
cursor=db.cursor()

import socket
timeout=1
a=1
socket.setdefaulttimeout(timeout)
from urllib2 import Request, urlopen, URLError, HTTPError

cursor.execute(’SELECT * FROM `stringf` LIMIT 20,5′)
numrows = int(cursor.rowcount)
# get and display one row at a time
x=20
for x in range(20,25):
row = cursor.fetchone()
print row[0], “–>”, row[1]
url=’http://www.’+row[1]+”
req = Request(url)
try:
response = urlopen(req)
except:
print ‘error’

else:
try:
the_page=response.read()
b=row[1]+’.txt’
f=open(b,’w')
f.write(the_page)
f.close()
except:
print ‘error’