Add statistics to WebUI: general stats, and stats for mirrors.

15 files changed, 155 insertions, 30 deletions

diff --git a/segget/connection.cpp b/segget/connection.cpp
index 5f5b1d2..77f18da 100644
--- a/segget/connection.cpp
+++ b/segget/connection.cpp
@@ -30,7 +30,6 @@
 long script_waiting_connection_num=-1;
 uint Tconnection::total_connections=0;
 Tconnection connection_array[MAX_CONNECTS];
-time_t prev_time;
 
 void init_connections(){
 	for (ulong connection_num=0; connection_num<MAX_CONNECTS; connection_num++){
@@ -133,14 +132,14 @@ void Tconnection::stop(CURLcode connection_result){
 		if (connection_result==0){
 			if (! segment->segment_verification_is_ok()){
 				connection_result=CURLE_READ_ERROR;
-				Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),0);
+				Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),0);
 				debug("curl_lies - there is a problem downloading segment");
 				error_log("curl_lies - there is a problem downloading segment");
 			}else{
-				Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),segment->segment_size);
+				Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),segment->segment_size);
 			}
 		}else{
-			Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),0);
+			Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),0);
 		}
 		segment->parent_distfile->active_connections_num--;
 
@@ -161,6 +160,7 @@ void Tconnection::stop(CURLcode connection_result){
 		gettimeofday(&now_time,NULL);
 
 		if (connection_result!=0){
+			stats.fails_counter++;
 			switch (network_array[network_num].network_mode){
 				case MODE_LOCAL:{
 	//				prnt_distfile->network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]=true;
@@ -228,7 +228,7 @@ void Tconnection::show_connection_progress(ulong time_diff){
 	try{
 		stats.total_bytes_per_last_interval+=bytes_per_last_interval;
 		ulong speed=(bytes_per_last_interval*1000)/time_diff;
-		ulong avg_speed=(total_dld_bytes*1000)/time_left_from(start_time);
+		ulong avg_speed=(total_dld_bytes*1000)/time_left_since(start_time);
 		string eta_string;
 		if (avg_speed==0){
 			eta_string=" ETA: inf";
@@ -266,10 +266,10 @@ void Tconnection::show_connection_progress(ulong time_diff){
 
 string Tconnection::get_html_connection_progress(){
 	try{
-		ulong time_diff=time_left_from(stats.previous_time);
+		ulong time_diff=time_left_since(stats.previous_time);
 //		stats.total_bytes_per_last_interval+=bytes_per_last_interval;
 		ulong speed=(bytes_per_last_interval*1000)/time_diff;
-		ulong avg_speed=(total_dld_bytes*1000)/time_left_from(start_time);
+		ulong avg_speed=(total_dld_bytes*1000)/time_left_since(start_time);
 		string eta_string;
 		if (avg_speed==0){
 			eta_string="inf";
@@ -318,7 +318,11 @@ string Tconnection::get_html_connection_progress(){
 			+"</tr></table>"
 //			+"&nbsp;"+toString(distfile_percent+unfinished_segments_distfile_percent)+"%"
 			+"&nbsp;"+toString(distfile_percent)+"%"
-			+"</td><td>"+segment->parent_distfile->name
+			+"</td><td>"
+				+"<table>"
+					+"<tr><td>"+segment->parent_distfile->name+"</td></tr>"
+					+"<tr><td>"+segment->url+"</td></tr>"
+				+"</table>"
 			+"</td><td align=center><table width=100 border=0 cellpadding=0 frame=void><tr>"
 				+((segment_percent>0)?"<td bgcolor=\"#00FF00\"><img src=\"/img/green_progress.jpg\" height=20 width="+toString(segment_percent)+"/></td>"
 					:"")
diff --git a/segget/connection.h b/segget/connection.h
index c1fcf6f..65b0ce1 100644
--- a/segget/connection.h
+++ b/segget/connection.h
@@ -76,7 +76,6 @@ class Tconnection{
 };
 
 extern long script_waiting_connection_num;
-extern time_t prev_time;
 extern Tconnection connection_array[MAX_CONNECTS];
 void init_connections();
 #endif
\ No newline at end of file
diff --git a/segget/distfile.cpp b/segget/distfile.cpp
index 552e1ef..2245616 100644
--- a/segget/distfile.cpp
+++ b/segget/distfile.cpp
@@ -609,9 +609,9 @@ uint Tdistfile::request_proxy_fetcher_network(uint network_priority){
 							case R_PF_REJECTED:break;
 							default:{
 								//if less then 30 secs left don't bother proxy-fetcher
-								if (2000>time_left_from(network_distfile_brokers_array[network_num].last_request_time)){
+								if (2000>time_left_since(network_distfile_brokers_array[network_num].last_request_time)){
 									debug("====================== Be more patient with proxy-fetcher - network#:"+toString(network_num)
-											+" because time left:"+toString(time_left_from(network_distfile_brokers_array[network_num].last_request_time)));
+											+" because time left:"+toString(time_left_since(network_distfile_brokers_array[network_num].last_request_time)));
 									return R_PF_BE_MORE_PATIENT;
 								}else{
 //						(network_array[best_proxy_fetcher_network_num].active_connections_num>network_array[network_num].active_connections_num)
diff --git a/segget/mirror.cpp b/segget/mirror.cpp
index 993cb90..5144e9b 100644
--- a/segget/mirror.cpp
+++ b/segget/mirror.cpp
@@ -69,7 +69,12 @@ string convert_to_coral_cdn_url(string url_address){
 
 double Tmirror::mirror_on_the_wall(){
 	try{
-		double criterion=honesty*1000000000*dld_time/dld_size;
+		double criterion;
+		if (dld_size>0){
+			criterion=honesty*1000000000*dld_time/dld_size;
+		}else{
+			criterion=honesty*1000000000*dld_time;
+		}
 		honesty=honesty*100/(100+settings.benchmark_oblivion);
 		return criterion;
 	}catch(...){
@@ -88,8 +93,14 @@ void Tmirror::start(){
 	}
 }
 
-void Tmirror::stop(ulong time, uint size){
+void Tmirror::stop(ulong time, ulong size){
 	try{
+		if (size==0){
+			failed_downloads++;
+		}else{
+			successful_downloads++;
+		};
+
 		dld_time+=time/1000;
 		dld_size+=size;
 		honesty=1;
diff --git a/segget/mirror.h b/segget/mirror.h
index 37d8653..1cdca2e 100644
--- a/segget/mirror.h
+++ b/segget/mirror.h
@@ -37,20 +37,24 @@ using namespace std;
 
 class Tmirror{
 	private:
-		uint active_num;
 	public:
+		uint active_num;
 		string url;
 		ulong	dld_time;
 		ulong	dld_size;
+		ulong failed_downloads;
+		ulong successful_downloads;
 		double	honesty;
 		Tmirror():
 			active_num(0),
 			url(""),
 			dld_time(0),
-			dld_size(1),
+			dld_size(0),
+			failed_downloads(0),
+			successful_downloads(0),
 			honesty(1){};
 		void	start();
-		void	stop(ulong time, uint size);
+		void	stop(ulong time, ulong size);
 		double mirror_on_the_wall();
 		uint get_active_num(){return active_num;};
 };
diff --git a/segget/scriptserver.cpp b/segget/scriptserver.cpp
index d257745..1632c0d 100644
--- a/segget/scriptserver.cpp
+++ b/segget/scriptserver.cpp
@@ -240,7 +240,7 @@ bool run_user_python_script(uint connection_num){
 		struct timeval user_script_start_time;
 		gettimeofday(&user_script_start_time,NULL);
 		
-		while(1000>time_left_from(user_script_start_time)) {
+		while(1000>time_left_since(user_script_start_time)) {
 			int fd;
 			int nread;
 			testfds = readfds;
diff --git a/segget/segget.cpp b/segget/segget.cpp
index 6a0d1c4..d13d731 100644
--- a/segget/segget.cpp
+++ b/segget/segget.cpp
@@ -198,7 +198,7 @@ int download_pkgs(){
 		struct timeval prev_connection_activation_cycle_time;
 		while (keep_running_flag) {
 			// Use free connections to download segments connections
-			if (1000>time_left_from(prev_connection_activation_cycle_time)){
+			if (1000>time_left_since(prev_connection_activation_cycle_time)){
 				debug("Not enough time left to start connection activation cycle");
 				sleep(1);
 			}else{
@@ -296,7 +296,7 @@ int download_pkgs(){
 }
 void *refresh_tui_screen(void * ){
 	while (true){
-		ulong time_diff_msecs=time_left_from(stats.previous_time);
+		ulong time_diff_msecs=time_left_since(stats.previous_time);
 		if (time_diff_msecs >= settings.current_speed_time_interval_msecs){
 			show_progress(time_diff_msecs);
 		};
@@ -381,7 +381,12 @@ int routine(){
 		signal(SIGABRT,segget_exit);//If program aborts go to assigned function "segget_exit".
 		signal(SIGTERM,segget_exit);//If program terminates go to assigned function "segget_exit".
 		signal(SIGINT,segget_exit);//If program terminates go to assigned function "segget_exit".
-		prev_time=time((time_t *)NULL);
+		try{
+			gettimeofday(&stats.segget_start_time,NULL);
+//			stats.prev_time=time((time_t *)NULL);
+		}catch(...){
+			error_log("Error in stats.cpp: reset_previous_time()");
+		}
 		try{
 			//load settings
 				settings.init();
diff --git a/segget/segment.cpp b/segget/segment.cpp
index bfbdc5c..82be817 100644
--- a/segget/segment.cpp
+++ b/segget/segment.cpp
@@ -189,8 +189,8 @@ void show_progress(double time_diff){
 			//if connection is not NULL
 			if (connection_array[con_num].active){
 				connection_array[con_num].show_connection_progress(time_diff);
-				if (time_left_from(connection_array[con_num].start_time)>0){
-					stats.avg_total_speed+=(connection_array[con_num].total_dld_bytes*1000)/time_left_from(connection_array[con_num].start_time);
+				if (time_left_since(connection_array[con_num].start_time)>0){
+					stats.avg_total_speed+=(connection_array[con_num].total_dld_bytes*1000)/time_left_since(connection_array[con_num].start_time);
 				}
 			}
 		}
diff --git a/segget/stats.h b/segget/stats.h
index 99f2ffa..b27971c 100644
--- a/segget/stats.h
+++ b/segget/stats.h
@@ -34,12 +34,13 @@
 using namespace std;
 
 class Tstats{
-	private:
+	public:
 		ulong dld_size;
 		ulong dld_distfiles_count;
 		uint total_size;
-	public:
 		ulong total_bytes_per_last_interval;
+//		time_t prev_time;
+		struct timeval segget_start_time;
 		struct timeval previous_time;
 		double last_time_interval;
 		ulong avg_total_speed;
@@ -48,11 +49,14 @@ class Tstats{
 		ulong segments_count;
 		ulong dld_segments_count;
 		ulong active_connections_counter;
+		ulong fails_counter;
 		Tstats():
 			dld_size(0),
 			dld_distfiles_count(0),
 			total_size(0),
 			total_bytes_per_last_interval(0),
+//			prev_time(),
+			segget_start_time(),
 			previous_time(),
 			last_time_interval(1),
 			avg_total_speed(0),
@@ -60,7 +64,8 @@ class Tstats{
 			distfiles_count(0),
 			segments_count(0),
 			dld_segments_count(0),
-			active_connections_counter(0)
+			active_connections_counter(0),
+			fails_counter(0)
 			{};
 		void inc_dld_size(ulong more_bytes){ dld_size+=more_bytes;};
 		ulong get_dld_size(){return dld_size;};
diff --git a/segget/str.cpp b/segget/str.cpp
index 592e0cd..e7b974e 100644
--- a/segget/str.cpp
+++ b/segget/str.cpp
@@ -51,6 +51,11 @@ string toString(bool t){
 	s << t;
 	return s.str();
 }
+string toString(double t){
+	stringstream s;
+	s << t;
+	return s.str();
+}
 /*
 template<typename T> string toString(T t){
 	stringstream s;
diff --git a/segget/str.h b/segget/str.h
index 6580fad..a118cea 100644
--- a/segget/str.h
+++ b/segget/str.h
@@ -44,6 +44,7 @@ string toString(int t);
 string toString(ulong t);
 string toString(long t);
 string toString(bool t);
+string toString(double t);
 string field(string prefix,ulong t, int width);
 
 #endif
\ No newline at end of file
diff --git a/segget/ui_server.cpp b/segget/ui_server.cpp
index 6ecf525..75de62c 100644
--- a/segget/ui_server.cpp
+++ b/segget/ui_server.cpp
@@ -284,14 +284,20 @@ string Tui_server::get_header(string title){
 			+"<tr align=center>"
 				+"<td><a href=\"connections\"><img src=\"/img/connections.jpg\" alt=\"Segments\" height=50 width=50/></a></td>"
 				+"<td><a href=\"distfiles\"><img src=\"/img/distfiles.png\" alt=\"Distfiles\" height=50 width=50/></a></td>"
+				+"<td><a href=\"stats\"><img src=\"/img/stats.jpg\" alt=\"Stats\" height=50 width=50/></a></td>"
+				+"<td><a href=\"mirrors_stats\"><img src=\"/img/mirrors.jpg\" alt=\"mirrors\" height=50 width=50/></a></td>"
 				+"<td><a href=\"log\"><img src=\"/img/log.png\" alt=\"Log\" height=50 width=50/></a></td>"
 				+"<td><a href=\"errors_log\"><img src=\"/img/errors_log.jpg\" alt=\"Errors log\" height=50 width=50/></a></td>"
+				+"<td><a href=\"rss\"><img src=\"/img/rss.jpg\" alt=\"Log\" height=50 width=50/></a></td>"
 			+"</tr>"
 			+"<tr align=center>"
 				+"<td><a href=\"connections\">Connections</a></td>"
 				+"<td><a href=\"distfiles\">Distfiles</a></td>"
+				+"<td><a href=\"stats\">Stats</a></td>"
+				+"<td><a href=\"mirrors_stats\">Mirrors</a></td>"
 				+"<td><a href=\"log\">Log</a></td>"
 				+"<td><a href=\"errors_log\">Errors log</a></td>"
+				+"<td><a href=\"rss\">RSS</a></td>"
 		+"</tr>"
 		+"</table>"
 		+"<h1>"+title+"</h1>";
@@ -302,15 +308,90 @@ string Tui_server::get_footer(){
 	return "</body></html>";
 }
 
+string Tui_server::get_stats(){
+	ulong total_progress;
+	if (stats.total_size>0){
+		total_progress=(stats.total_size-stats.dld_size)*100/stats.total_size;
+	}else total_progress=0;
+	string stats_str=
+		(string)"<table border=\"1\">"
+			+"<tr><td>Up time</td><td align=right>"+secsToString(time_left_since(stats.segget_start_time)/1000)+"</td></tr>"
+			+"<tr><th colspan=3>Distfiles</td></tr>"
+				+"<tr><td>Downloaded distfiles</td><td align=right>"+toString(stats.dld_distfiles_count)+"</td></tr>"
+				+"<tr><td>Bytes downloaded</td><td align=right>"+toString(stats.dld_size)+"</td></tr>"
+				+"<tr><td>Total progress</td><td align=right>"+toString(total_progress)+"%</td></tr>"
+				+"<tr><td>Distfiles left to download</td><td align=right>"+toString(stats.distfiles_count-stats.dld_distfiles_count)+"</td></tr>"
+				+"<tr><td>Bytes left to download</td><td align=right>"+toString(stats.total_size-stats.dld_size)+"</td></tr>"
+				+"<tr><td>Distfiles total</td><td align=right>"+toString(stats.distfiles_count)+"</td></tr>"
+				+"<tr><td>Bytes total</td><td align=right>"+toString(stats.total_size)+"</td></tr>"
+			+"<tr><th colspan=3>Segments</td></tr>"
+				+"<tr><td>Downloaded segments</td><td align=right>"+toString(stats.dld_segments_count)+"</td></tr>"
+				+"<tr><td>Segmentss left to download</td><td align=right>"+toString(stats.segments_count-stats.dld_segments_count)+"</td></tr>"
+				+"<tr><td>Segments total</td><td align=right>"+toString(stats.segments_count)+"</td></tr>"
+			+"<tr><th colspan=3>Connections</td></tr>"
+				+"<tr><td>Failed connections</td><td align=right>"+toString(stats.fails_counter)+"</td></tr>"
+				+"<tr><td>AVG speed of active connections</td><td align=right>"+speedToString(stats.avg_total_speed)+"</td></tr>"
+//				+"<tr><td>AVG speed since segget start</td><td align=right>"+speedToString(stats.dld_size*1000/time_left_since(stats.segget_start_time))+"</td></tr>"
+//				+"<tr><td>AVG speed since segget start except idle times</td><td align=right>"+speedToString(stats.avg_total_speed)+"</td></tr>"
+		+"</table>";
+	return stats_str;
+}
+
+
+string Tui_server::get_mirrors_stats(){
+	string stats_str=(string)"<table border=\"1\" width=100%>"
+			+"<tr><th>URL</td>"
+				+"<th>Active connections</th>"
+				+"<th>AVG speed per connection**</th>"
+				+"<th>Failed downloads</th>"
+				+"<th>Successful downloads</th>"
+				+"<th>Fail ratio</th>"
+				+"<th>Usage time**, sec</th>"
+				+"<th>Succesfuly downloaded bytes</th>"
+				+"<th>Honesty</th>"
+			+"</tr>";
+	map<string,Tmirror *>::iterator iter; 
+	for( iter = mirror_list.begin(); iter != mirror_list.end(); iter++ ) {
+		ulong avg_speed;
+		if ((iter->second->dld_size!=1) and (iter->second->dld_time>0)){
+			avg_speed=(iter->second->dld_size)/iter->second->dld_time;
+		}else avg_speed=0;
+
+		ulong fail_ratio;
+		if (iter->second->failed_downloads+iter->second->successful_downloads>0){
+			fail_ratio=(iter->second->failed_downloads)*100/(iter->second->failed_downloads+iter->second->successful_downloads);
+		}else fail_ratio=0;
+		stats_str=stats_str
+			+"<tr><td>"+iter->first+"</td>"
+				+"<td align=right>"+toString(iter->second->active_num)+"</td>"
+				+"<td align=right>"+speedToString(avg_speed)+"</td>"
+				+"<td align=right>"+toString(iter->second->failed_downloads)+"</td>"
+				+"<td align=right>"+toString(iter->second->successful_downloads)+"</td>"
+				+"<td align=right>"+toString(fail_ratio)+"%</td>"
+				+"<td align=right>"+toString(iter->second->dld_time)+"</td>"
+				+"<td align=right>"+toString(iter->second->dld_size)+"</td>"
+				+"<td align=right width=15%>"+toString(iter->second->honesty)+"</td>"
+			+"</tr>";
+	}
+	stats_str=stats_str+"</table>";
+
+	stats_str=stats_str+"<br><p>** NOTE: When a mirror has N simultaneous connections "
+		+"\"Usage time\" will be increasing N times faster, therefore \"AVG speed per connection\" will "
+		+"be smaller than average outgoing rate provided by this mirror to your host "
+		+"(which is a summary speed of the simultaneous connections to this mirror)</p>";
+	return stats_str;
+}
+
+
 string Tui_server::get_connections_info(){
 	try{
 		string result=(string)"<center>"
 			+"<h3>Active connections: "+toString(stats.active_connections_counter)+"/"+toString(settings.max_connections)+"</h1>"
 		+"<table border=1 width=100%>"
 			+"<tr>"
-				+"<th rowspan=2>Distfile progress</th>"
+				+"<th rowspan=2 width=110>Distfile progress</th>"
 				+"<th rowspan=2>Distfile name</th>"
-				+"<th rowspan=2>Segment progress</th>"
+				+"<th rowspan=2 width=110>Segment progress</th>"
 				+"<th rowspan=2>Segment #</th>"
 				+"<th rowspan=2>Try</th>"
 				+"<th colspan=2>Network</th>"
@@ -439,6 +520,14 @@ void Tui_server::serve_browser(uint fd, string msg){
 			send_to_fd(fd,get_footer());
 		}else if ((uri=="/rss") or (uri=="/rss.rss") or (uri=="/rss.xml")){
 			send_to_fd(fd,get_rss_info());
+		}else if (uri=="/stats"){
+			ui_server.send_to_fd(fd,get_header("Stats"));
+			ui_server.send_to_fd(fd,get_stats());
+			ui_server.send_to_fd(fd,get_footer());
+		}else if (uri=="/mirrors_stats"){
+			ui_server.send_to_fd(fd,get_header("Mirrors stats"));
+			ui_server.send_to_fd(fd,get_mirrors_stats());
+			ui_server.send_to_fd(fd,get_footer());
 		}else if (uri=="/log"){
 			ui_server.send_to_fd(fd,get_header("Log"));
 			ui_server.send_to_fd(fd,"<center><table border=\"1\" width=\"100%\">");
@@ -460,7 +549,7 @@ void Tui_server::serve_browser(uint fd, string msg){
 			ui_server.send_to_fd(fd,"<center><table border=\"1\" width=\"100%\">");
 				debug("Sending to client distfiles_num:"+toString(request_server_pkg.Pdistfile_list.size()));
 				ui_server.send_to_fd(fd,(string)"<tr>"
-				+"</th><th rowspan=2>Progress"
+				+"</th><th rowspan=2 width=110>Progress"
 				+"</th><th rowspan=2>Name"
 				+"</th><th rowspan=2>Status"
 				+"</th><th colspan=2>Segments"
diff --git a/segget/ui_server.h b/segget/ui_server.h
index 1de8761..cdc0931 100644
--- a/segget/ui_server.h
+++ b/segget/ui_server.h
@@ -69,6 +69,8 @@ class Tui_server{
 		string serve_browser_distfile_progress(Tdistfile * a_distfile);
 		string get_connections_info();
 		string get_rss_info();
+		string get_stats();
+		string get_mirrors_stats();
 		void serve_browser(uint fd, string msg);
 };
 
diff --git a/segget/utils.cpp b/segget/utils.cpp
index 9d80cb4..851484f 100644
--- a/segget/utils.cpp
+++ b/segget/utils.cpp
@@ -26,7 +26,7 @@
 
 #include "utils.h"
 
-ulong time_left_from(timeval from_time){
+ulong time_left_since(timeval from_time){
 	try{
 		timeval now_time;
 		gettimeofday(&now_time,NULL);
diff --git a/segget/utils.h b/segget/utils.h
index dfd5eff..ae95c29 100644
--- a/segget/utils.h
+++ b/segget/utils.h
@@ -33,7 +33,7 @@
 
 using namespace std;
 
-ulong time_left_from(timeval from_time);
+ulong time_left_since(timeval from_time);
 string secsToString(ulong secs);
 string speedToString(ulong dld_bytes, ulong time_left);
 string speedToString(ulong speed);