#!/usr/bin/perl
#
#Program Name:  getyahoo.pl
#
#Description: 	Extract securities data from Yahoo for export to spreadsheet.
#		We read a list of URLs that give the data in historical format.
#
#  The following is an example of a Yahoo URL which requests a comma delimited file.
#http://chart.yahoo.com/table.csv?a=1&b=1&c=2003&d=1&e=15&f=2003&s=aapl&y=0&g=d&ignore=.csv
#
#we read each URL and do the following:
#
# Each stock output is in reverse chronological order with column headers,
# e.g.,
# 	Date,Open,High,Low,Close,Volume
# 	14-Feb-03,14.61,14.72,14.35,14.67,4344600
# 	13-Feb-03,14.41,14.64,14.24,14.54,3723100
# 	12-Feb-03,14.27,14.60,14.27,14.39,4083700
#
#We then close the files and reopen the output file as input
#in order to put the stock symbol as the first column of each line,
#after we filter out the title lines.
#This makes output to HTML easier in the format program.
#
#
# e.g.,
#	aapl,14-Feb-03,14.61,14.72,14.35,14.67,4344600
# 	aapl,13-Feb-03,14.41,14.64,14.24,14.54,3723100
# 	aapl,12-Feb-03,14.27,14.60,14.27,14.39,4083700
#
#
#**********************************************************************
#Author: Noe Dinnerstein, BodhiSoftWare, Inc.
#Date:	2003/08/28
#********************************************************************************

require LWP::UserAgent;
require LWP::RobotUA;
use URI::URL;

#delete old output files
unlink "webread.tmp";
unlink "webread.dat";


#open driver file
open (STOCKS,"<stockdata.txt") or die "Failed open input stockdata.txt";

while (<STOCKS>) {
	$url=$_;
	chomp($url);	#trim new-line from URL
	RecSub($url);	#get URL as one input stream with new-lines
	FinalOut();	#put in output file with symbol on each line
			#to allow parsing in HTML creation program
}

close STOCKS;

exit(0);

#================================================
sub FinalOut {
#open final output file,concatenating to previous ones in this run
	open (SDAT,">>webread.dat") or die "Failed open output webread.dat";
	close RPT;
#now reopen RPT to reformat with stock symbol
	open (RPT,"<webread.tmp") or die "Failed open input webread.tmp";

	while (<RPT>) {
	 formatDat();
	}

	close RPT;	#close temp file
	close SDAT;	#close drive

	unlink 'webread.tmp';	#delete temp file
}

#==================================================
sub RecSub
#Get contents of a URL
{
#open output file
	open (RPT,">webread.tmp") or die "Failed open output webread.tmp";

        my($NewUrl) = $_[0]; #get the parameter passed to the subroutine
        local $url = $NewUrl ; #assign to local variable

        #create instance of LWP web robot User Agent to pass to HTTP request
	#these are dummy parameters. If you wish, you can identify yourself
	my $ua = new LWP::RobotUA 'my-robot/1.0', 'MyDomain@MyDomain.com';
        $ua->delay(0);

	#create instance of HTTP get request for URL.
        $request = new HTTP::Request('GET', $url);
        $response = $ua->request($request);
        $res = $response->content;

	#parse out stock symbol to pass to extract file
	$symloc = index($url,"s=")+2;
	$symend = index($url,"&y=");
	$symlen = $symend - $symloc;
	$sym = substr($url,$symloc,$symlen);

	print RPT "$sym\n";   #print stock symbol
	print RPT "$res";	#print the yahoo output

}

sub formatDat {
	if ($_ =~ m/,/) {     #filter out single entry lines (no comma )
       	 	if ($_ =~ m/([0-9])/g) {    #filter out title  (no numbers)
			print SDAT "$sym,$_";		 
		}
	}
}
#==================================================