Python tweepy.Stream() Examples

def streamAPI(auth):
        # We instantiate our listener
        listener = StreamListener()
        # We start the streamer with the OAuth object and the listener
        streamer = tweepy.Stream(auth=auth, listener=listener)
        # We define the terms of which we want to track
        targetTerms = ['python']
        # We start the streamer, passing it our trackTerms
        streamer.filter(track=targetTerms) 

def get_streaming_data(self):
        twitter_stream = Stream(self.auth, twitter_listener(num_tweets_to_grab=self.num_tweets_to_grab, retweet_count = self.retweet_count, stats = self.stats, get_tweet_html = self.get_tweet_html ))
        try:
            twitter_stream.sample()
        except Exception as e:
            print(e.__doc__)

        lang, top_lang, top_tweets = self.stats.get_stats()
        print(Counter(lang))
        print(Counter(top_lang))
        print(len(top_tweets))

        self.c.execute("INSERT INTO lang_data VALUES (?,?, DATETIME('now'))", (str(list(Counter(lang).items())), str(list(Counter(top_lang).items()))))

        for t in top_tweets:
            self.c.execute("INSERT INTO twit_data VALUES (?, DATETIME('now'))", (t,))

        self.conn.commit() 

def __init__(self, auth, listener, retry_count, logger, min_http_delay=5,
		max_http_delay=320, min_http_420_delay=60, min_tcp_ip_delay=0.5,
		max_tcp_ip_delay=16, **options):

		self.logger = logger
		self.logger.info('COMPLIENT STREAM: Initializing complient stream...')
		self.min_http_delay = min_http_delay
		self.max_http_delay = max_http_delay
		self.min_tcp_ip_delay = min_tcp_ip_delay
		self.max_tcp_ip_delay = max_tcp_ip_delay
		self.running = False
		self.retry_count = retry_count
		self.auth = auth

		#Twitter sends a keep-alive every twitter_keepalive seconds
		self.twitter_keepalive = 30

		#Add a couple seconds more wait time.
		self.twitter_keepalive += 2.0

		self.sleep_time = 0

		#logging.info('COMPLIANT STREAM: Initializing compliant stream...')

		tweepy.Stream.__init__(self, auth, listener, secure=True, **options) 

def initialize():

	output = TwitterStreamListener()

	# setup Twitter API connection details
	twitter_auth = OAuthHandler( TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET )
	twitter_auth.set_access_token( TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_TOKEN_SECRET )
	
	# connect to Twitter Streaming API
	twitter_stream = Stream( twitter_auth, output )

	# filter tweets using track, follow and/or location parameters
	# https://dev.twitter.com/streaming/reference/post/statuses/filter
	twitter_stream.filter(track=[ TWITTER_HASHTAG ])


# def cleanup():
# 	twitter_stream.disconnect() 

def main(args):
    if args.debug:
        logger.setLevel(logging.DEBUG)

    auth = tweepy.OAuthHandler(args.consumer_key, args.consumer_secret)
    auth.set_access_token(args.access_token, args.access_token_secret)
    api = tweepy.API(auth, wait_on_rate_limit=True)
    screen_name = api.me().screen_name

    if args.classifier == 'mock':
        classifier = classifiers.MockClassifier()
    elif args.classifier == 'local':
        classifier = classifiers.URLClassifier(classifiers.ImageClassifier(args.dataset_path, INPUT_SHAPE))
    elif args.classifier == 'remote':
        classifier = classifiers.RemoteClassifier(args.remote_endpoint)

    stream = tweepy.Stream(auth=auth, listener=ReplyToTweet(screen_name, classifier, api, args.silent))
    logger.info('Listening as {}'.format(screen_name))
    stream.userstream(track=[screen_name]) 

def start_stream(self):
        """Starts a stream with teh current tracking terms"""

        tracking_terms = self.term_checker.tracking_terms()

        if len(tracking_terms) > 0 or self.unfiltered:
            # we have terms to track, so build a new stream
            self.stream = tweepy.Stream(self.auth, self.listener,
                                        stall_warnings=True,
                                        timeout=90,
                                        retry_count=self.retry_count)

            if len(tracking_terms) > 0:
                logger.info("Starting new twitter stream with %s terms:", len(tracking_terms))
                logger.info("  %s", repr(tracking_terms))
                
                # Launch it in a new thread
                self.stream.filter(track=tracking_terms, async=True, languages=self.languages)
            else:
                logger.info("Starting new unfiltered stream")
                self.stream.sample(async=True, languages=self.languages) 

def start(schema, logger):

        listener = TwitterListener(config.DATASET_ID, config.TABLE_ID, logger=logger)
        auth = tweepy.OAuthHandler(config.CONSUMER_KEY, config.CONSUMER_SECRET)
        auth.set_access_token(config.ACCESS_TOKEN, config.ACCESS_TOKEN_SECRET)

        while True:

            logger.info("Connecting to Twitter stream")

            stream = None

            try:

                stream = tweepy.Stream(auth, listener, headers = {"Accept-Encoding": "deflate, gzip"})

                # Choose stream: filtered or sample
                stream.sample()
                # stream.filter(track=TwitterListener.TRACK_ITEMS)

            except:

                logger.exception("Unexpected error");

                if stream:
                    stream.disconnect()

                time.sleep(60) 

def __init__(self,
                 pipeline,
                 batch_size=1000,
                 consumer_key=settings.CONSUMER_TOKEN,
                 consumer_secret=settings.CONSUMER_SECRET,
                 acces_token=settings.ACCESS_TOKEN,
                 access_secret=settings.ACCESS_SECRET,):
        self.auth = tweepy.OAuthHandler(consumer_key=consumer_key, consumer_secret=consumer_secret)
        self.auth.set_access_token(acces_token, access_secret)
        self.stream = tweepy.Stream(auth=self.auth, listener=Listener(pipeline, batch_size=batch_size))
        logging.basicConfig(filename='log_twitter.txt', level=logging.DEBUG) 

def main():
    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    api = tweepy.API(auth)

    listener = TwitterStreamListener()
    twitter_stream = tweepy.Stream(auth=api.auth, listener=listener)
    twitter_stream.filter(track=TRACKS, languages=['en'], async=True) 

def __init__(self, phrases):
        """Use static method 'get_listener' instead of constructing directly"""
        _log.info("Creating listener for [%s]" % ", ".join(phrases))
        self.__phrases = phrases
        self.__subscriptions = set()
        self.__tweets = [None] * self.__max_size

        # listen for tweets in a background thread using the 'async' keyword
        auth = OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_token, access_token_secret)
        self.__stream = Stream(auth, listener=self)
        self.__stream.filter(track=phrases, async=True)
        self.__connected = True 

def auth_streamer(self):
        stream = tweepy.Stream(auth=self.twweeter_obj.api.auth,
                               listener=self.stream_listener_ob)
        return stream

    # listen for tweets containing a specific word or hashtag \
    #(a phrase might work too) 

def stream(self):
        """
        tweepy.Stream object.
        :return:
        """
        if self._stream is None:
            self._stream = Stream(
                self.auth, self,
                **self._stream_options)
        return self._stream 

def update_stream(self):
        """
        Restarts the stream with the current list of tracking terms.
        """

        need_to_restart = False

        # If we think we are running, but something has gone wrong in the streaming thread
        # Restart it.
        if self.stream is not None and not self.stream.running:
            logger.warning("Stream exists but isn't running")
            self.listener.error = False
            self.listener.streaming_exception = None
            need_to_restart = True

        # Check if the tracking list has changed
        if self.term_checker.check():
            logger.info("Terms have changed")
            need_to_restart = True

        # If we aren't running and we are allowing unfiltered streams
        if self.stream is None and self.unfiltered:
            need_to_restart = True

        if not need_to_restart:
            return

        logger.info("Restarting stream...")
        
        # Stop any old stream
        self.stop_stream()

        # Start a new stream
        self.start_stream() 

def start_streaming(self, callback):
        """Starts streaming tweets and returning data to the callback."""

        self.twitter_listener = TwitterListener(
            callback=callback, logs_to_cloud=self.logs_to_cloud)
        twitter_stream = Stream(self.twitter_auth, self.twitter_listener)

        self.logs.debug("Starting stream.")
        twitter_stream.filter(follow=[TRUMP_USER_ID])

        # If we got here because of an API error, raise it.
        if self.twitter_listener and self.twitter_listener.get_error_status():
            raise Exception("Twitter API error: %s" %
                            self.twitter_listener.get_error_status()) 

def main():
    # parser
    parser = argparse.ArgumentParser()
    parser.add_argument('--config', type=str, required=False,
                        help='config file path')
    parser.add_argument('--db', type=str, required=True,
                        help='path to sqlite3 db')
    parser.add_argument('--lang', type=str, required=True,
                        help='target language')
    args = parser.parse_args()

    listener = QueueListener(args)
    stream = Stream(listener.auth, listener)
    print("Listening...\n")
    delay = 0.25
    try:
        while True:
            try:
                stream.sample()
            except KeyboardInterrupt:
                print('Stopped')
                return
            except urllib3.exceptions.ProtocolError as e:
                print("Incomplete read", e)
            except urllib3.exceptions.ReadTimeoutError as e:
                print("Read Timeout", e)
            except (socket.error, http.client.HTTPException):
                print("HTTP error waiting for a few seconds")
                time.sleep(delay)
                delay += 0.25
    finally:
        stream.disconnect() 

def _start_stream(self, tweet_list: list, api: tw.API) -> None:
        try:
            stream_start = TweetListener(api, self.bot)
            self.mystream = tw.Stream(api.auth, stream_start, daemon=True)
            fake_task = functools.partial(self.mystream.filter, follow=tweet_list, is_async=True)
            task = self.bot.loop.run_in_executor(None, fake_task)
            try:
                await asyncio.wait_for(task, timeout=5)
            except asyncio.TimeoutError:
                log.info("Timeout opening tweet stream.")
                pass
        except Exception:
            log.error("Error starting stream", exc_info=True) 

def triggertweets():
	key = request.args['keyword']
	if key==None:
		return ('No keyword Present')
	max_tweets = int(request.args.get('max_tweets',10))

	#print(key)
	l = StdOutListener(key,max_tweets)
	stream = Stream(api.auth, l)
	stream.filter(track=[key])
	# stream.filter(track=[key], async=True)
	return jsonify({"trigger":"started"})

#--------------New Function-------------------- 

def get_streaming_data(self):
        twitter_stream = Stream(self.auth, twitter_listener(num_tweets_to_grab=self.num_tweets_to_grab, retweet_count = self.retweet_count, stats = self.stats, get_tweet_html = self.get_tweet_html ))
        try:
            twitter_stream.sample()
        except Exception as e:
            print(e.__doc__)

        lang, top_lang, top_tweets = self.stats.get_stats()
        print(Counter(lang))
        print(Counter(top_lang))
        print(top_tweets) 

def get_streaming_data(self):
        twitter_stream = Stream(self.auth, twitter_listener(num_tweets_to_grab=self.num_tweets_to_grab, retweet_count = self.retweet_count))
        try:
            twitter_stream.sample()
        except Exception as e:
            print(e.__doc__) 

def listen():
    start_twitter()
    try:
        app_exfiltrate.log_message('info', "[twitter] Listening for DMs...")
        stream = Stream(auth, StdOutListener())
        stream.userstream()
    except Exception, e:
        app_exfiltrate.log_message(
            'warning', "[twitter] Couldn't listen for Twitter DMs".format(e)) 

def streamer_entrypoint():
    twitter_stream = tweepy.Stream(twitter.auth, AccountListener())
    twitter_stream.filter(follow=[str(twitter.get_id(k[0])) for k in configuration.get_accounts()],
                          async=False) 

def main():
    global api

    try:
        auth = OAuthHandler(CONSUMER_TOKEN, CONSUMER_SECRET)
        auth.secure = True
        auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

        api = API(auth)
        stream = Stream(auth, StdOutListener())
        stream.userstream()

    except BaseException as e:
        print("Error in main()", e) 

def start(self):
        stream = tweepy.Stream(self.auth, self)
        stream.filter(track=['Bitcoin', 'Ethereum', 'Crypto']) 

def initialize():
    with open('data/config.json') as config_data:
        config = json.load(config_data)

    auth = tweepy.OAuthHandler(config['consumer_key'], config['consumer_secret'])
    auth.set_access_token(config['access_token'], config['access_token_secret'])
    api = tweepy.API(auth)

    stream = TwitterStreamListener()
    twitter_stream = tweepy.Stream(auth = api.auth, listener=stream)
    twitter_stream.filter(track=['iphone'], async=True) 

def _start_user_stream(self, retweet_replies_to_ids: List[int]):
        auth = self._auth()
        # Setup reply stream for handling mentions and DM
        self._user_stream = tweepy.Stream(auth, TwitterReplyListener(self, self._credentials, retweet_replies_to_ids))
        self._user_stream.userstream(async=True) 

def get_tweets(duik, old_duik):
    this_app = AppOpenLSH.get_or_insert('KeyOpenLSH')
    auth = tweepy.OAuthHandler(this_app.twitter_consumer_key, this_app.twitter_consumer_secret)
    auth.set_access_token(this_app.twitter_access_token_key, this_app.twitter_access_token_secret)
    api = tweepy.API(auth)
    listen = TwitterStatusListener(duik, old_duik, auth)
    stream = tweepy.Stream(auth, listen)

    try:
        stream.sample()
    except tweepy.TweepError:
        logging.error("error with streaming api")
        stream.disconnect()
    return (listen.tweets) 

def get_tweets(users, outfile, stop_num):
    auth, api = login()
    twitter_stream = Stream(auth, MyListener(outfile, stop_num))
    twitter_stream.filter(follow=users, async=True) 

def get_tweets(my_locations, outfile, stop_num):
    auth, api = login()
    twitter_stream = Stream(auth, MyListener(outfile, stop_num))
    # Bounding boxes for geo-locations
    # http://boundingbox.klokantech.com/
    # Online-Tool to create boxes (c+p as raw CSV):
    twitter_stream.filter(locations=my_locations, async=True) 

def stream_tweets(self, fetched_tweets_filename, hash_tag_list):
        # This handles Twitter authetification and the connection to Twitter Streaming API
        listener = TwitterListener(fetched_tweets_filename)
        auth = self.twitter_autenticator.authenticate_twitter_app() 
        stream = Stream(auth, listener)

        # This line filter Twitter Streams to capture data by the keywords: 
        stream.filter(track=hash_tag_list)


# # # # TWITTER STREAM LISTENER # # # # 

def stream_tweets(self, fetched_tweets_filename, hash_tag_list):
        # This handles Twitter authetification and the connection to Twitter Streaming API
        listener = TwitterListener(fetched_tweets_filename)
        auth = self.twitter_autenticator.authenticate_twitter_app() 
        stream = Stream(auth, listener)

        # This line filter Twitter Streams to capture data by the keywords: 
        stream.filter(track=hash_tag_list)


# # # # TWITTER STREAM LISTENER # # # #