From 661d296c7cfb76c794f9438dbae72027e8d4a128 Mon Sep 17 00:00:00 2001 From: Kostyantyn Ovechko Date: Sun, 8 Aug 2010 18:25:41 +0300 Subject: Add statistics to WebUI: general stats, and stats for mirrors. --- segget/connection.cpp | 20 ++++++----- segget/connection.h | 1 - segget/distfile.cpp | 4 +-- segget/mirror.cpp | 15 ++++++-- segget/mirror.h | 10 ++++-- segget/scriptserver.cpp | 2 +- segget/segget.cpp | 11 ++++-- segget/segment.cpp | 4 +-- segget/stats.h | 11 ++++-- segget/str.cpp | 5 +++ segget/str.h | 1 + segget/ui_server.cpp | 95 +++++++++++++++++++++++++++++++++++++++++++++++-- segget/ui_server.h | 2 ++ segget/utils.cpp | 2 +- segget/utils.h | 2 +- 15 files changed, 155 insertions(+), 30 deletions(-) diff --git a/segget/connection.cpp b/segget/connection.cpp index 5f5b1d2..77f18da 100644 --- a/segget/connection.cpp +++ b/segget/connection.cpp @@ -30,7 +30,6 @@ long script_waiting_connection_num=-1; uint Tconnection::total_connections=0; Tconnection connection_array[MAX_CONNECTS]; -time_t prev_time; void init_connections(){ for (ulong connection_num=0; connection_numsegment_verification_is_ok()){ connection_result=CURLE_READ_ERROR; - Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),0); + Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),0); debug("curl_lies - there is a problem downloading segment"); error_log("curl_lies - there is a problem downloading segment"); }else{ - Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),segment->segment_size); + Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),segment->segment_size); } }else{ - Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),0); + Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),0); } segment->parent_distfile->active_connections_num--; @@ -161,6 +160,7 @@ void Tconnection::stop(CURLcode connection_result){ gettimeofday(&now_time,NULL); if (connection_result!=0){ + stats.fails_counter++; switch (network_array[network_num].network_mode){ case MODE_LOCAL:{ // prnt_distfile->network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]=true; @@ -228,7 +228,7 @@ void Tconnection::show_connection_progress(ulong time_diff){ try{ stats.total_bytes_per_last_interval+=bytes_per_last_interval; ulong speed=(bytes_per_last_interval*1000)/time_diff; - ulong avg_speed=(total_dld_bytes*1000)/time_left_from(start_time); + ulong avg_speed=(total_dld_bytes*1000)/time_left_since(start_time); string eta_string; if (avg_speed==0){ eta_string=" ETA: inf"; @@ -266,10 +266,10 @@ void Tconnection::show_connection_progress(ulong time_diff){ string Tconnection::get_html_connection_progress(){ try{ - ulong time_diff=time_left_from(stats.previous_time); + ulong time_diff=time_left_since(stats.previous_time); // stats.total_bytes_per_last_interval+=bytes_per_last_interval; ulong speed=(bytes_per_last_interval*1000)/time_diff; - ulong avg_speed=(total_dld_bytes*1000)/time_left_from(start_time); + ulong avg_speed=(total_dld_bytes*1000)/time_left_since(start_time); string eta_string; if (avg_speed==0){ eta_string="inf"; @@ -318,7 +318,11 @@ string Tconnection::get_html_connection_progress(){ +"" // +" "+toString(distfile_percent+unfinished_segments_distfile_percent)+"%" +" "+toString(distfile_percent)+"%" - +""+segment->parent_distfile->name + +"" + +"" + +"" + +"" + +"
"+segment->parent_distfile->name+"
"+segment->url+"
" +"" +((segment_percent>0)?"" :"") diff --git a/segget/connection.h b/segget/connection.h index c1fcf6f..65b0ce1 100644 --- a/segget/connection.h +++ b/segget/connection.h @@ -76,7 +76,6 @@ class Tconnection{ }; extern long script_waiting_connection_num; -extern time_t prev_time; extern Tconnection connection_array[MAX_CONNECTS]; void init_connections(); #endif \ No newline at end of file diff --git a/segget/distfile.cpp b/segget/distfile.cpp index 552e1ef..2245616 100644 --- a/segget/distfile.cpp +++ b/segget/distfile.cpp @@ -609,9 +609,9 @@ uint Tdistfile::request_proxy_fetcher_network(uint network_priority){ case R_PF_REJECTED:break; default:{ //if less then 30 secs left don't bother proxy-fetcher - if (2000>time_left_from(network_distfile_brokers_array[network_num].last_request_time)){ + if (2000>time_left_since(network_distfile_brokers_array[network_num].last_request_time)){ debug("====================== Be more patient with proxy-fetcher - network#:"+toString(network_num) - +" because time left:"+toString(time_left_from(network_distfile_brokers_array[network_num].last_request_time))); + +" because time left:"+toString(time_left_since(network_distfile_brokers_array[network_num].last_request_time))); return R_PF_BE_MORE_PATIENT; }else{ // (network_array[best_proxy_fetcher_network_num].active_connections_num>network_array[network_num].active_connections_num) diff --git a/segget/mirror.cpp b/segget/mirror.cpp index 993cb90..5144e9b 100644 --- a/segget/mirror.cpp +++ b/segget/mirror.cpp @@ -69,7 +69,12 @@ string convert_to_coral_cdn_url(string url_address){ double Tmirror::mirror_on_the_wall(){ try{ - double criterion=honesty*1000000000*dld_time/dld_size; + double criterion; + if (dld_size>0){ + criterion=honesty*1000000000*dld_time/dld_size; + }else{ + criterion=honesty*1000000000*dld_time; + } honesty=honesty*100/(100+settings.benchmark_oblivion); return criterion; }catch(...){ @@ -88,8 +93,14 @@ void Tmirror::start(){ } } -void Tmirror::stop(ulong time, uint size){ +void Tmirror::stop(ulong time, ulong size){ try{ + if (size==0){ + failed_downloads++; + }else{ + successful_downloads++; + }; + dld_time+=time/1000; dld_size+=size; honesty=1; diff --git a/segget/mirror.h b/segget/mirror.h index 37d8653..1cdca2e 100644 --- a/segget/mirror.h +++ b/segget/mirror.h @@ -37,20 +37,24 @@ using namespace std; class Tmirror{ private: - uint active_num; public: + uint active_num; string url; ulong dld_time; ulong dld_size; + ulong failed_downloads; + ulong successful_downloads; double honesty; Tmirror(): active_num(0), url(""), dld_time(0), - dld_size(1), + dld_size(0), + failed_downloads(0), + successful_downloads(0), honesty(1){}; void start(); - void stop(ulong time, uint size); + void stop(ulong time, ulong size); double mirror_on_the_wall(); uint get_active_num(){return active_num;}; }; diff --git a/segget/scriptserver.cpp b/segget/scriptserver.cpp index d257745..1632c0d 100644 --- a/segget/scriptserver.cpp +++ b/segget/scriptserver.cpp @@ -240,7 +240,7 @@ bool run_user_python_script(uint connection_num){ struct timeval user_script_start_time; gettimeofday(&user_script_start_time,NULL); - while(1000>time_left_from(user_script_start_time)) { + while(1000>time_left_since(user_script_start_time)) { int fd; int nread; testfds = readfds; diff --git a/segget/segget.cpp b/segget/segget.cpp index 6a0d1c4..d13d731 100644 --- a/segget/segget.cpp +++ b/segget/segget.cpp @@ -198,7 +198,7 @@ int download_pkgs(){ struct timeval prev_connection_activation_cycle_time; while (keep_running_flag) { // Use free connections to download segments connections - if (1000>time_left_from(prev_connection_activation_cycle_time)){ + if (1000>time_left_since(prev_connection_activation_cycle_time)){ debug("Not enough time left to start connection activation cycle"); sleep(1); }else{ @@ -296,7 +296,7 @@ int download_pkgs(){ } void *refresh_tui_screen(void * ){ while (true){ - ulong time_diff_msecs=time_left_from(stats.previous_time); + ulong time_diff_msecs=time_left_since(stats.previous_time); if (time_diff_msecs >= settings.current_speed_time_interval_msecs){ show_progress(time_diff_msecs); }; @@ -381,7 +381,12 @@ int routine(){ signal(SIGABRT,segget_exit);//If program aborts go to assigned function "segget_exit". signal(SIGTERM,segget_exit);//If program terminates go to assigned function "segget_exit". signal(SIGINT,segget_exit);//If program terminates go to assigned function "segget_exit". - prev_time=time((time_t *)NULL); + try{ + gettimeofday(&stats.segget_start_time,NULL); +// stats.prev_time=time((time_t *)NULL); + }catch(...){ + error_log("Error in stats.cpp: reset_previous_time()"); + } try{ //load settings settings.init(); diff --git a/segget/segment.cpp b/segget/segment.cpp index bfbdc5c..82be817 100644 --- a/segget/segment.cpp +++ b/segget/segment.cpp @@ -189,8 +189,8 @@ void show_progress(double time_diff){ //if connection is not NULL if (connection_array[con_num].active){ connection_array[con_num].show_connection_progress(time_diff); - if (time_left_from(connection_array[con_num].start_time)>0){ - stats.avg_total_speed+=(connection_array[con_num].total_dld_bytes*1000)/time_left_from(connection_array[con_num].start_time); + if (time_left_since(connection_array[con_num].start_time)>0){ + stats.avg_total_speed+=(connection_array[con_num].total_dld_bytes*1000)/time_left_since(connection_array[con_num].start_time); } } } diff --git a/segget/stats.h b/segget/stats.h index 99f2ffa..b27971c 100644 --- a/segget/stats.h +++ b/segget/stats.h @@ -34,12 +34,13 @@ using namespace std; class Tstats{ - private: + public: ulong dld_size; ulong dld_distfiles_count; uint total_size; - public: ulong total_bytes_per_last_interval; +// time_t prev_time; + struct timeval segget_start_time; struct timeval previous_time; double last_time_interval; ulong avg_total_speed; @@ -48,11 +49,14 @@ class Tstats{ ulong segments_count; ulong dld_segments_count; ulong active_connections_counter; + ulong fails_counter; Tstats(): dld_size(0), dld_distfiles_count(0), total_size(0), total_bytes_per_last_interval(0), +// prev_time(), + segget_start_time(), previous_time(), last_time_interval(1), avg_total_speed(0), @@ -60,7 +64,8 @@ class Tstats{ distfiles_count(0), segments_count(0), dld_segments_count(0), - active_connections_counter(0) + active_connections_counter(0), + fails_counter(0) {}; void inc_dld_size(ulong more_bytes){ dld_size+=more_bytes;}; ulong get_dld_size(){return dld_size;}; diff --git a/segget/str.cpp b/segget/str.cpp index 592e0cd..e7b974e 100644 --- a/segget/str.cpp +++ b/segget/str.cpp @@ -51,6 +51,11 @@ string toString(bool t){ s << t; return s.str(); } +string toString(double t){ + stringstream s; + s << t; + return s.str(); +} /* template string toString(T t){ stringstream s; diff --git a/segget/str.h b/segget/str.h index 6580fad..a118cea 100644 --- a/segget/str.h +++ b/segget/str.h @@ -44,6 +44,7 @@ string toString(int t); string toString(ulong t); string toString(long t); string toString(bool t); +string toString(double t); string field(string prefix,ulong t, int width); #endif \ No newline at end of file diff --git a/segget/ui_server.cpp b/segget/ui_server.cpp index 6ecf525..75de62c 100644 --- a/segget/ui_server.cpp +++ b/segget/ui_server.cpp @@ -284,14 +284,20 @@ string Tui_server::get_header(string title){ +"" +"" +"" + +"" + +"" +"" +"" + +"" +"" +"" +"" +"" + +"" + +"" +"" +"" + +"" +"" +"
\"Segments\"\"Distfiles\"\"Stats\"\"mirrors\"\"Log\"\"Errors\"Log\"
ConnectionsDistfilesStatsMirrorsLogErrors logRSS
" +"

"+title+"

"; @@ -302,15 +308,90 @@ string Tui_server::get_footer(){ return ""; } +string Tui_server::get_stats(){ + ulong total_progress; + if (stats.total_size>0){ + total_progress=(stats.total_size-stats.dld_size)*100/stats.total_size; + }else total_progress=0; + string stats_str= + (string)"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" +// +"" +// +"" + +"
Up time"+secsToString(time_left_since(stats.segget_start_time)/1000)+"
Distfiles
Downloaded distfiles"+toString(stats.dld_distfiles_count)+"
Bytes downloaded"+toString(stats.dld_size)+"
Total progress"+toString(total_progress)+"%
Distfiles left to download"+toString(stats.distfiles_count-stats.dld_distfiles_count)+"
Bytes left to download"+toString(stats.total_size-stats.dld_size)+"
Distfiles total"+toString(stats.distfiles_count)+"
Bytes total"+toString(stats.total_size)+"
Segments
Downloaded segments"+toString(stats.dld_segments_count)+"
Segmentss left to download"+toString(stats.segments_count-stats.dld_segments_count)+"
Segments total"+toString(stats.segments_count)+"
Connections
Failed connections"+toString(stats.fails_counter)+"
AVG speed of active connections"+speedToString(stats.avg_total_speed)+"
AVG speed since segget start"+speedToString(stats.dld_size*1000/time_left_since(stats.segget_start_time))+"
AVG speed since segget start except idle times"+speedToString(stats.avg_total_speed)+"
"; + return stats_str; +} + + +string Tui_server::get_mirrors_stats(){ + string stats_str=(string)"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +""; + map::iterator iter; + for( iter = mirror_list.begin(); iter != mirror_list.end(); iter++ ) { + ulong avg_speed; + if ((iter->second->dld_size!=1) and (iter->second->dld_time>0)){ + avg_speed=(iter->second->dld_size)/iter->second->dld_time; + }else avg_speed=0; + + ulong fail_ratio; + if (iter->second->failed_downloads+iter->second->successful_downloads>0){ + fail_ratio=(iter->second->failed_downloads)*100/(iter->second->failed_downloads+iter->second->successful_downloads); + }else fail_ratio=0; + stats_str=stats_str + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +"" + +""; + } + stats_str=stats_str+"
URL" + +"Active connectionsAVG speed per connection**Failed downloadsSuccessful downloadsFail ratioUsage time**, secSuccesfuly downloaded bytesHonesty
"+iter->first+""+toString(iter->second->active_num)+""+speedToString(avg_speed)+""+toString(iter->second->failed_downloads)+""+toString(iter->second->successful_downloads)+""+toString(fail_ratio)+"%"+toString(iter->second->dld_time)+""+toString(iter->second->dld_size)+""+toString(iter->second->honesty)+"
"; + + stats_str=stats_str+"

** NOTE: When a mirror has N simultaneous connections " + +"\"Usage time\" will be increasing N times faster, therefore \"AVG speed per connection\" will " + +"be smaller than average outgoing rate provided by this mirror to your host " + +"(which is a summary speed of the simultaneous connections to this mirror)

"; + return stats_str; +} + + string Tui_server::get_connections_info(){ try{ string result=(string)"
" +"

Active connections: "+toString(stats.active_connections_counter)+"/"+toString(settings.max_connections)+"

" +"" +"" - +"" + +"" +"" - +"" + +"" +"" +"" +"" @@ -439,6 +520,14 @@ void Tui_server::serve_browser(uint fd, string msg){ send_to_fd(fd,get_footer()); }else if ((uri=="/rss") or (uri=="/rss.rss") or (uri=="/rss.xml")){ send_to_fd(fd,get_rss_info()); + }else if (uri=="/stats"){ + ui_server.send_to_fd(fd,get_header("Stats")); + ui_server.send_to_fd(fd,get_stats()); + ui_server.send_to_fd(fd,get_footer()); + }else if (uri=="/mirrors_stats"){ + ui_server.send_to_fd(fd,get_header("Mirrors stats")); + ui_server.send_to_fd(fd,get_mirrors_stats()); + ui_server.send_to_fd(fd,get_footer()); }else if (uri=="/log"){ ui_server.send_to_fd(fd,get_header("Log")); ui_server.send_to_fd(fd,"
Distfile progressDistfile progressDistfile nameSegment progressSegment progressSegment #TryNetwork
"); @@ -460,7 +549,7 @@ void Tui_server::serve_browser(uint fd, string msg){ ui_server.send_to_fd(fd,"
"); debug("Sending to client distfiles_num:"+toString(request_server_pkg.Pdistfile_list.size())); ui_server.send_to_fd(fd,(string)"" - +"
Progress" + +"Progress" +"Name" +"Status" +"Segments" diff --git a/segget/ui_server.h b/segget/ui_server.h index 1de8761..cdc0931 100644 --- a/segget/ui_server.h +++ b/segget/ui_server.h @@ -69,6 +69,8 @@ class Tui_server{ string serve_browser_distfile_progress(Tdistfile * a_distfile); string get_connections_info(); string get_rss_info(); + string get_stats(); + string get_mirrors_stats(); void serve_browser(uint fd, string msg); }; diff --git a/segget/utils.cpp b/segget/utils.cpp index 9d80cb4..851484f 100644 --- a/segget/utils.cpp +++ b/segget/utils.cpp @@ -26,7 +26,7 @@ #include "utils.h" -ulong time_left_from(timeval from_time){ +ulong time_left_since(timeval from_time){ try{ timeval now_time; gettimeofday(&now_time,NULL); diff --git a/segget/utils.h b/segget/utils.h index dfd5eff..ae95c29 100644 --- a/segget/utils.h +++ b/segget/utils.h @@ -33,7 +33,7 @@ using namespace std; -ulong time_left_from(timeval from_time); +ulong time_left_since(timeval from_time); string secsToString(ulong secs); string speedToString(ulong dld_bytes, ulong time_left); string speedToString(ulong speed); -- cgit v1.2.3-65-gdbad