Using Streaming API with User Input on Node.js

[Jemzy2013]

Hello all.

I am trying to develop a system on Node.js using the Twit module for the Twitter Streaming API. I can successfully start a stream using static input and it will continuously print out tweets but when I try and do the same thing using a POST method it will only print tweets every 30-60 seconds at a time instead of constantly. It could be where my code is located in the script but I am not entirely sure why it’s doing this. Here is my code:

var Twit = require('twit');
var express = require('express');
var app = express();
var http = require('http').Server(app);
var io = require('socket.io')(http);
var querystring = require('querystring');
var utils = require('util');
var finalSearch;

var T = new Twit({
    consumer_key:         ''
  , consumer_secret:      ''
  , access_token:         ''
  , access_token_secret:  ''
});

// THIS WORKS BUT IS STATIC BUT I NEED DYNAMIC SEARCH TERMS

//stream = T.stream('statuses/filter', { track: [ 'twitter' , 'feed' ] });
//
//stream.on('tweet', function(tweet) {
//    io.emit('tweet', tweet);
//});

app.get('/style.css', function(req, res) {
	res.sendFile(__dirname + '/css/style.css');
});

app.get('/', function(req, res) {
	res.sendFile(__dirname + '/index.html');
});

app.post('/', function(req, res) {
	var search = '';
	req.on('data', function(chunk) {
		search += chunk.toString();
	});

	req.on('end', function() {
		res.writeHead(200, "OK", {'Content-Type': 'text/html'});

		var decodedSearch = querystring.parse(search);
		var inspectSearch = utils.inspect(decodedSearch);
		finalSearch = inspectSearch.substring(13, (inspectSearch.length - 2));
		finalSearch = finalSearch.replace(", ", " , ");
		
		console.log(finalSearch);
		var stream = T.stream('statuses/filter', { track: finalSearch });
		
		stream.on('tweet', function(tweet) {
			io.emit('tweet', tweet);
		});
				
		stream.on('connect', function(request) {
			console.log('[CONNECTING] Twitter Stream connection attempted.');
		});
				
		stream.on('connected', function(response) {
			console.log('[CONNECTED] Twitter Stream connection successful.');
		});

		res.end();
	});
});

http.listen(3000, function(){
	console.log('Server listening on port 3000.');
});

So that is my code including the working streaming code at the top. Any suggestions are appreciated.

Thanks.

[boxnumber03]

Try adding handlers for the ‘connected’ and ‘reconnect’ events. Like this:

stream.on('connected', function (res) {
	console.log('stream connected (' + res.statusCode + ')');
});
stream.on('reconnect', function (req, res, interval) {
	console.log('stream reconnecting in ' + interval + ' (' + res.statusCode + ')');
});

If you find the status code is 420, then you are connecting too frequently.

[Jemzy2013]

Wow. You were right. I am getting 420 status codes. Do you have any ideas how to fix it? Since I know about it now, I will try some things in the meantime whilst awaiting a reply but I will update this reply if I fix it.Here’s the response code I get from the console:

Server listening on port 3000.
'beer'
[CONNECTING] Twitter Stream connection attempted.
[CONNECTED] Twitter Stream connection successful.
stream connected (420)
stream reconnecting in 60000 (420)

Where ‘beer’ is just the search term I entered.

Thanks heaps for that. That helps a lot. Don’t know why I didn’t test the codes before.

EDIT: I tried making it only call the stream once like so:

var oldSearch = "";

req.on('end', function() {
		res.writeHead(200, "OK", {'Content-Type': 'text/html'});

		var decodedSearch = querystring.parse(search);
		var inspectSearch = utils.inspect(decodedSearch);
		var finalSearch = inspectSearch.substring(13, (inspectSearch.length - 2));
		finalSearch = finalSearch.replace(", ", " , ");
		
		if (oldSearch != finalSearch) {
			oldSearch = finalSearch;
			var stream = T.stream('statuses/filter', { track: finalSearch });
	
			stream.on('tweet', function(tweet) {
				console.log('[TWEET] Tweet received.');
				io.emit('tweet', tweet);
			});
	
			stream.on('connect', function(request) {
				console.log('[CONNECTING] Twitter Stream connection attempted.');
			});
			
			stream.on('connected', function (res) {
				console.log('[CONNECTED] Stream connected (' + res.statusCode + ')');
			});
			stream.on('reconnect', function (req, res, interval) {
				console.log('[RECONNECTING] Stream reconnecting in ' + interval + ' (' + res.statusCode + ')');
			});	
		}

		res.end();
	});

And it does the same thing as before and print out a tweet after a long period of time. Here is the console output:

[CONNECTING] Twitter Stream connection attempted.
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[TWEET] Tweet received.
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)
[CONNECTED] Stream connected (200)

Bit strange…

[boxnumber03]

Twit does use the “back off” strategy described in the Twitter docs. So, I’m not sure what there is to fix. But, you probably would like to avoid getting rate-limited with 420 errors. For that you have to prevent the user from connecting too frequently. What “too frequently” is is not stated as far as I know. Either way, it means the user must wait, which probably doesn’t help you.

[Jemzy2013]

Not quite sure what I did but I fixed it. This was my code in the end:

req.on('end', function() {
		res.writeHead(200, "OK", {'Content-Type': 'text/html'});		
		var decodedSearch = querystring.parse(search);
		
		if (oldSearch != search) {
			oldSearch = search;
			var stream = T.stream('statuses/filter', { track: decodedSearch['terms[]'] });
	
			stream.on('tweet', function(tweet) {
				unirest.post("https://twinword-sentiment-analysis.p.mashape.com/analyze/")
				.header("X-Mashape-Key", "")
				.header("Content-Type", "application/x-www-form-urlencoded")
				.field("text", tweet.text)
				.end(function (result) {
					io.emit('tweet', tweet);
					io.emit('sentiment', result.body);
				});
			});
		}

		res.end();
	});

Somehow it worked. Lol. Just did some refactoring of the code and it decided to work.