crawler (python)

#author: Maria Beatriz Vaz

import urlparse
import urllib
import codecs
from bs4 import BeautifulSoup
import psycopg2
import sys

try:
    conn = psycopg2.connect("dbname='tweet_ic' user='postgres' host='localhost' port='5432' password='8001007'")
    print "Connected to database"
except:
    print "I am unable to connect to the database"

cur=conn.cursor()

urls = ["https://twitter.com/search?q=%23marcocivil%20since%3A2014-02-03%20until%3A2014-05-23&src=typd"]
visited = [urls[0]]

while len(urls) > 0:
try:
htmltext = urllib.urlopen(urls[0]).read()
except:
print urls[0]

soup = BeautifulSoup(htmltext)
print soup.get_text().encode("utf-8")

visited.append(urls[0])


for tag in soup.findAll('div',  { "class" : "content" }) :
for tweet in tag.findAll('p', {"class" : "js-tweet-text tweet-text"}):
print ">>>>nova tag"
tweet_text = tweet.get_text().encode("utf-8")
print tweet_text
cur.execute(""" INSERT INTO teste_old (tweets) VALUES  (%(tweets)s)""", ({"tweets" : tweet.get_text().encode("utf-8")}))
conn.commit()

print ">>>visitou: " + urls[0]
urls.pop(0)

if conn:
conn.close()



Já consigo pegar o texto de tweets de uma determinada página. Agora preciso pegar, pelo menos, o usuário e a data daquele tweet.

Preciso saber também como faz para o programa descer a página.

Com o site de pesquisa https://twitter.com/search?q=%23marcocivil%20since%3A2014-02-03%20until%3A2014-05-23&src=typd
só capturei 19 tweets, ´pois é o que aparece até abaixarmos mais a tela.