Login | Register For Free | Help
Search for: (Advanced)

Mailing List Archive: Wikipedia: Mediawiki-CVS

SVN: [115480] trunk/wikistats/MetricsReporting

 

 

Wikipedia mediawiki-cvs RSS feed   Index | Next | Previous | View Threaded


ezachte at svn

Jun 4, 2012, 4:23 PM

Post #1 of 1 (302 views)
Permalink
SVN: [115480] trunk/wikistats/MetricsReporting

https://www.mediawiki.org/wiki/Special:Code/MediaWiki/115480

Revision: 115480
Author: ezachte
Date: 2012-06-04 23:23:03 +0000 (Mon, 04 Jun 2012)
Log Message:
-----------
Files used at stat1:/a/wikistats/analytics. Current use is to prep data for new report card in wikilytics*.csv files.

Added Paths:
-----------
trunk/wikistats/MetricsReporting/analytics/
trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepBinariesData.pl
trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepComscoreData.pl
trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepLanguageNames.pl
trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepPageViews.pl
trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutput.pl
trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutputCore.pl
trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutputMisc.pl
trunk/wikistats/MetricsReporting/analytics/README
trunk/wikistats/MetricsReporting/analytics/analytics_create_and_load_from_csv.txt
trunk/wikistats/MetricsReporting/analytics/analytics_generate_csv_files.sh
trunk/wikistats/MetricsReporting/analytics/analytics_generate_csv_files_log.txt
trunk/wikistats/MetricsReporting/analytics/analytics_new.sh
trunk/wikistats/MetricsReporting/analytics/analytics_refresh_from_csv.txt
trunk/wikistats/MetricsReporting/analytics/analytics_test.sh
trunk/wikistats/MetricsReporting/analytics/analytics_test_query.txt
trunk/wikistats/MetricsReporting/analytics/analytics_upd.sh
trunk/wikistats/MetricsReporting/analytics/analytics_upd_log.txt
trunk/wikistats/MetricsReporting/analytics/backup.sh
trunk/wikistats/MetricsReporting/analytics/use_db_analytics.txt

Added: trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepBinariesData.pl
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepBinariesData.pl (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepBinariesData.pl 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,124 @@
+#!/usr/local/bin/perl
+
+ use Getopt::Std ;
+
+ &ParseArguments ;
+
+ print "Write file '$file_csv_out'\n" ;
+ open CSV_OUT, '>', $file_csv_out ;
+
+ foreach $project (qw (wb wk wn wp wq ws wv wx))
+ { &ReadStatisticsPerBinariesExtension ($project) ; }
+
+ close CSV_OUT ;
+
+ print "\n\nReady\n\n" ;
+ exit ;
+
+sub ParseArguments
+{
+ my @options ;
+ getopt ("io", \%options) ;
+
+ die ("Specify input folder as: -i path") if (! defined ($options {"i"})) ;
+ die ("Specify output folder as: -o path'") if (! defined ($options {"o"})) ;
+
+ $path_in = $options {"i"} ;
+ $path_out = $options {"o"} ;
+
+ die "Input folder '$path_in' does not exist" if (! -d $path_in) ;
+ die "Output folder '$path_out' does not exist" if (! -d $path_out) ;
+
+ # tests only
+ # $path_in = "C:/@ Wikimedia/# Out Bayes" ;
+ # $path_out = "C:/analytics" ; # "w:/@ report card/data" ;
+
+ print "Input folder: $path_in\n" ;
+ print "Output folder: $path_out\n" ;
+ print "\n" ;
+
+ $file_csv_out = "$path_out/analytics_in_binaries.csv" ;
+}
+
+
+sub ReadStatisticsPerBinariesExtension
+{
+ my $project = shift ;
+ my $file_csv_in = "$path_in/csv_$project/StatisticsPerBinariesExtension.csv" ;
+ $yyyymm_hi = -1 ;
+
+ if (! -e $file_csv_in)
+ { die "Input file '$file_csv_in' not found" ; }
+
+
+ print "Read '$file_csv_in'\n" ;
+ open CSV_IN, '<', $file_csv_in ;
+
+ $language_prev = '' ;
+ while ($line = <CSV_IN>)
+ {
+ chomp $line ;
+ next if $line !~ /,.*?,/ ;
+
+ ($language,$date,$data) = split (',', $line, 3) ;
+
+ # for each wiki first line shows ext names, no tcounts
+ if ($date eq "00/0000")
+ {
+ if ($language_prev ne '')
+ { &WriteMonthlyData ($project, $language_prev) ; }
+ $language_prev = $language ;
+
+ undef %ext_name ;
+ undef %ext_ndx ;
+ undef %ext_cnt ;
+ undef %months ;
+
+ @exts = split (',', $data) ;
+ $ndx = 0 ;
+ foreach $ext (@exts)
+ {
+ $ext_name {$ndx} = $ext ;
+ $ext_ndx {$ext} = $ndx ;
+ $ndx ++ ;
+ }
+ next ;
+ }
+
+ ($month,$year) = split ('\/', $date) ;
+ $yyyymm = sprintf ("%04d-%02d", $year, $month) ;
+ if ($yyyymm gt $yyyymm_hi)
+ { $yyyymm_hi = $yyyymm ; }
+ $months {$yyyymm}++ ;
+
+ @counts = split (',', $data) ;
+ $ndx = 0 ;
+ foreach $count (@counts)
+ {
+ $ext_cnt {$yyyymm}{$ext_name {$ndx}} = $count ;
+ $ndx ++ ;
+ }
+ }
+ &WriteMonthlyData ($project, $language_prev) ;
+
+ close CSV_IN ;
+}
+
+sub WriteMonthlyData
+{
+ my ($project,$language) = @_ ;
+ # get sorted array of extensions, order by count for most recent month
+ %ext_cnt_yyyymm_hi = %{$ext_cnt {$yyyymm_hi}} ;
+ @ext_cnt_yyyymm_hi = (sort {$ext_cnt_yyyymm_hi {$b} <=> $ext_cnt_yyyymm_hi {$a}} keys %ext_cnt_yyyymm_hi) ;
+
+ foreach $month (sort keys %months)
+ {
+ $ndx = 0 ;
+ foreach $ext (@ext_cnt_yyyymm_hi)
+ {
+ print CSV_OUT "$project,$language,$month,$ext,${ext_cnt{$month}{$ext}}\n" ;
+ # print "$month,$ext,${ext_cnt{$month}{$ext_name {$ndx}}}\n" ;
+ last if (++ $ndx > 25) ;
+ }
+ }
+}

Added: trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepComscoreData.pl
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepComscoreData.pl (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepComscoreData.pl 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,677 @@
+#!/usr/bin/perl
+
+# Copyright (C) 2011 Wikimedia Foundation
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 2
+# as published by the Free Software Foundation.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details, at
+# http://www.fsf.org/licenses/gpl.html
+
+# Author:
+# Erik Zachte, email ezachte [at] wikimedia
+
+# Functionality:
+# comScore data can be downloaded as csv file, which each contain 14 months history
+# This script uses these files to update 'master' csv files which contain all known history
+# Note: only entities which are already in master file will be updated!
+# Then it merges these master files into one csv file which can be loaded into analytics database
+# Data are: reach by region, unique visitors by region, unique visitors by web property
+
+# Parameters:
+# -m (required) folder with 'master' csv files (files with all known history)
+# -u (required) folder with 'update' csv files (files with lastest 14 months history, produced by comScore)
+
+# Output:
+# updated master csv files + merged and formatted csv for import in MySQL
+
+# http://svn.wikimedia.org/viewvc/mediawiki/trunk/wikistats/analytics/
+
+ use Getopt::Std ;
+ use Cwd;
+ use Time::Local ;
+
+
+ my $options ;
+ getopt ("imo", \%options) ;
+
+ $true = 1 ;
+ $false = 0 ;
+
+ $script_name = "AnalyticsPrepComscoreData.pl" ;
+ $script_version = "0.31" ;
+
+# EZ test only
+# $source = "comscore" ;
+# $server = "ez_test" ;
+# $generated = "2011-05-06 00:00:00" ;
+# $user = "ezachte" ;
+
+ $dir_in = $options {"i"} ;
+ $dir_upd = $options {"m"} ;
+ $dir_out = $options {"o"} ;
+ $mode = 'add';
+ if (defined $options {'r'})
+ { $mode = 'replace'; }
+
+ print "Mode is $mode (specify '-r' for replace)\n\n";
+
+ if (! -d "/a/") # EZ test machine
+ {
+ $dir_in = "W:/@ Report Card/Data" ;
+ $dir_upd = "W:/! Perl/AnalyticsDB" ;
+ $dir_out = "W:/! Perl/AnalyticsDB" ;
+ $mode = 'replace' ;
+ }
+
+ if ($dir_in eq '')
+ { Abort ("Specify folder for input file (new comScore data) '-i folder'") ; }
+ if ($dir_upd eq '')
+ { Abort ("Specify folder for master files (full history) as '-m folder'") ; }
+ if ($dir_out eq '')
+ { Abort ("Specify folder for output file '-o folder'") ; }
+
+ $file_comscore_reach_master = "history_comscore_reach_regions.csv" ;
+ $file_comscore_reach_update = "*reach*by*region*csv" ;
+ $file_comscore_uv_region_master = "history_comscore_UV_regions.csv" ;
+ $file_comscore_uv_region_update = "*UVs*by*region*csv" ;
+ $file_comscore_uv_property_master = "history_comscore_UV_properties.csv" ;
+ $file_comscore_uv_property_update = "*UV*trend*csv" ;
+
+ $layout_csv_reach = 1 ;
+ $layout_csv_regions = 2 ;
+ $layout_csv_properties = 3 ;
+
+ print "Directories:\n" .
+ "Input (new comScore data): '$dir_in'\n".
+ "Master files (full history): '$dir_upd'\n" .
+ "Output (database feed): '$dir_out'\n\n" ;
+
+ %region_codes = (
+ "Europe"=>"EU",
+ "North America"=>"NA",
+ "Latin America"=>"LA",
+ "World-Wide" => "W",
+ "Middle East - Africa" => "MA",
+ "Asia Pacific"=> "AS",
+ "United States" => "US",
+ "India" => "I",
+ "China" => "C"
+ ) ;
+
+ foreach $region_name (keys %region_codes)
+ { $region_names {$region_codes {$region_name}} = $region_name ; }
+
+ @months_short = qw "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec" ;
+
+ &ReadMasterComscoreDataReachPerRegion ($file_comscore_reach_master, $file_comscore_reach_update, "%.1f", 1, $layout_csv_reach) ;
+ %reach_region_code = %data ;
+
+ &ReadMasterComscoreDataVisitorsPerRegion ($file_comscore_uv_region_master, $file_comscore_uv_region_update, "%.0f", 1000, $layout_csv_regions) ;
+ %visitors_region_code = %data ;
+
+ &ReadMasterComscoreDataVisitorsPerProperty ($file_comscore_uv_property_master, $file_comscore_uv_property_update, "%.0f", 1000, $layout_csv_properties) ;
+ %visitors_web_property = %data ;
+
+ &ReadDictionaryRegions ;
+
+ &WriteDataAnalyticsMySQL ;
+ &WriteDataAnalyticsDygraph ;
+
+ print "\nReady\n\n" ;
+ exit ;
+
+sub ReadMasterComscoreDataReachPerRegion
+{
+ my ($file_comscore_master, $file_comscore_updates, $precision, $layout_csv) = @_ ;
+
+ print "ReadMasterComscoreDataReachPerRegion\n\n" ;
+
+ undef %months ;
+ undef %data ;
+ undef @regions ;
+
+ open IN, '<', "$dir_upd/$file_comscore_master" ;
+
+ $lines = 0 ;
+ while ($line = <IN>)
+ {
+ chomp $line ;
+ $line =~ s/\r//g ;
+
+ ($yyyymm,@data) = split (',', $line) ;
+ if ($lines++ == 0)
+ {
+ @regions = @data ;
+ print "Regions found: " . (join ',', @regions) . "\n";
+ next ;
+ }
+
+ $field_ndx = 0 ;
+ foreach (@data)
+ {
+ $region = $regions [$field_ndx] ;
+ $region_code = $region_codes {$region} ;
+
+ $data = $data [$field_ndx] ;
+ if ($data eq '')
+ { $data = '0' ; }
+ $months {$yyyymm} ++ ;
+ $data {"$yyyymm,$region_code"} = $data ;
+ # print "Old data $yyyymm,$region_code = $data\n" ;
+ $field_ndx++ ;
+ }
+ }
+ close IN ;
+
+ my $updates_found = &UpdateMasterFileFromRecentComscoreData ($file_comscore_master, $file_comscore_updates, 1, $layout_csv, @regions) ;
+ return if ! $updates_found ;
+
+ rename "$dir_upd/$file_comscore_master", "$dir_upd/$file_comscore_master.~" ;
+ open OUT, '>', "$dir_upd/$file_comscore_master" ;
+
+ $line_out = "yyyymm" ;
+ foreach $region_name (@regions)
+ { $line_out .= ",$region_name" ; }
+ print OUT "$line_out" ;
+
+ foreach $yyyymm (sort {$b cmp $a} keys %months)
+ {
+ $line_out = "\n$yyyymm" ;
+ foreach $region_name (@regions)
+ {
+ $yyyymm_region_code = $yyyymm . ',' . $region_codes {$region_name} ;
+ $line_out .= "," . sprintf ($precision, $data {$yyyymm_region_code}) ;
+ }
+ print OUT "$line_out" ;
+ }
+
+ close OUT ;
+}
+
+sub ReadMasterComscoreDataVisitorsPerRegion
+{
+ my ($file_comscore_master, $file_comscore_updates, $precision, $multiplier, $layout_csv) = @_ ;
+
+ print "ReadMasterComscoreDataVisitorsPerRegion\n\n";
+
+ undef %months ;
+ undef %data ;
+ undef @regions ;
+
+ open IN, '<', "$dir_upd/$file_comscore_master" ;
+
+ $lines = 0 ;
+ $metric = 'unique_visitors' ;
+ while ($line = <IN>)
+ {
+ chomp $line ;
+ $line =~ s/\r//g ;
+ $line = &GetNumberOnly ($line) ;
+
+ next if $line !~ /(?:yyyymm|\d\d\d\d-\d\d)/ ;
+
+ ($yyyymm,@data) = split (',', $line) ;
+
+ if ($lines++ == 0)
+ {
+ @regions = @data ;
+ print "Regions found: " . (join ',', @regions) . "\n";
+ next ;
+ }
+
+ $field_ndx = 0 ;
+ foreach (@data)
+ {
+ $region = $regions [$field_ndx] ;
+ $region_code = $region_codes {$region} ;
+
+ $data = $data [$field_ndx] ;
+ if ($data eq '')
+ { $data = '0' ; }
+
+ # print "Old data $yyyymm,$region = $data\n" ;
+
+ $months {$yyyymm} ++ ;
+ $data {"$yyyymm,$region_code"} = $data ;
+
+ $field_ndx++ ;
+ }
+ }
+ close IN ;
+
+ my $updates_found = &UpdateMasterFileFromRecentComscoreData ($file_comscore_master, $file_comscore_updates, 1000, $layout_csv, @regions) ;
+ return if ! $updates_found ;
+
+ rename "$dir_upd/$file_comscore_master", "$dir_upd/$file_comscore_master.~" ;
+ open OUT, '>', "$dir_upd/$file_comscore_master" ;
+
+ $line_out = "yyyymm" ;
+ foreach $region_name (@regions)
+ { $line_out .= ",$region_name" ; }
+ print OUT "$line_out" ;
+
+ foreach $yyyymm (sort {$b cmp $a} keys %months)
+ {
+ $line_out = "\n$yyyymm" ;
+ foreach $region_name (@regions)
+ {
+ $yyyymm_region_code = $yyyymm . ',' . $region_codes {$region_name} ;
+ $line_out .= "," . sprintf ($precision, $data {$yyyymm_region_code}) ;
+ }
+ print OUT "$line_out" ;
+ }
+
+ close OUT ;
+}
+
+sub ReadMasterComscoreDataVisitorsPerProperty
+{
+ my ($file_comscore_master, $file_comscore_updates, $precision, $multiplier, $layout_csv) = @_ ;
+
+ print "ReadMasterComscoreDataVisitorsPerProperty\n\n";
+
+ undef %months ;
+ undef %data ;
+ undef @properties ;
+
+ open IN, '<', "$dir_upd/$file_comscore_master" ;
+
+ $lines = 0 ;
+ $metric = 'unique_visitors' ;
+ while ($line = <IN>)
+ {
+ chomp $line ;
+ $line =~ s/\r//g ;
+
+ ($yyyymm,@data) = split (',', $line) ;
+ if ($lines++ == 0)
+ { @properties = @data ; next ; }
+
+ $field_ndx = 0 ;
+ foreach (@data)
+ {
+ $property = $properties [$field_ndx] ;
+ $property =~ s/.*Yahoo.*/Yahoo/ ;
+ $property =~ s/,/&comma;/ ;
+ $data = $data [$field_ndx] ;
+ if ($data eq '')
+ { $data = '0' ; }
+
+ # print "Old data $yyyymm,$property = $data\n" ;
+
+ $months {$yyyymm} ++ ;
+ $data {"$yyyymm,$property"} = $data ;
+
+ $field_ndx++ ;
+ }
+ }
+ close IN ;
+
+ my $updates_found = &UpdateMasterFileFromRecentComscoreData ($file_comscore_master, $file_comscore_updates, 1000, $layout_csv, @properties) ;
+ return if ! $updates_found ;
+
+ rename "$dir_upd/$file_comscore_master", "$dir_upd/$file_comscore_master.~" ;
+ open OUT, '>', "$dir_upd/$file_comscore_master" ;
+
+ $line_out = "yyyymm" ;
+ foreach $property (@properties)
+ { $line_out .= ",$property" ; }
+ print OUT "$line_out" ;
+
+ foreach $yyyymm (sort {$b cmp $a} keys %months)
+ {
+ $line_out = "\n$yyyymm" ;
+ foreach $property (@properties)
+ {
+ $yyyymm_property = "$yyyymm,$property" ;
+ $line_out .= "," . sprintf ($precision, $data {$yyyymm_property}) ;
+ }
+ print OUT "$line_out" ;
+ }
+
+ close OUT ;
+}
+
+sub ReadDictionaryRegions
+{
+ open CSV_IN, '<', "$dir_in/wikilytics_in_dictionary_regions.csv" ;
+
+ $line = <CSV_IN> ; # skip titles
+ while ($line = <CSV_IN>)
+ {
+ chomp $line ;
+ ($language, $code, $name) = split (',', $line) ;
+
+ next if $language ne 'en' ;
+
+ $name_region {$code} = $name ;
+ }
+
+ close CSV_IN ;
+}
+
+sub UpdateMasterFileFromRecentComscoreData
+{
+ my ($file_comscore_master, $file_comscore_updates, $multiplier, $layout_csv, @white_list) = @_ ;
+
+ print "UpdateMasterFileFromRecentComscoreData\n\n";
+
+ undef %white_list ;
+ undef %not_white_listed ;
+
+ print "White list: ". (join (',', @white_list)) . "\n\n";
+
+ foreach $id (@white_list)
+ { $white_list {$id} = $true ; }
+
+ if (! -e "$dir_upd/$file_comscore_master")
+ { Abort ("File $file_comscore_master not found!") ; }
+
+ $age_all = -M "$dir_upd/$file_comscore_master" ;
+ print "Latest comscore master file is " . sprintf ("%.0f", $age_all) . " days old: '$file_comscore_master'\n" ;
+
+ my $cwd = getcwd ;
+ chdir $dir_in ;
+
+ @files = glob($file_comscore_updates) ;
+ $min_age_upd = 999999 ;
+ $file_comscore_updates_latest = '' ;
+ foreach $file (@files)
+ {
+ $age = -M $file ;
+ if ($age < $min_age_upd)
+ {
+ $min_age_upd = $age ;
+ $file_comscore_updates_latest = $file ;
+ }
+ }
+ print "Latest comscore update file is " . sprintf ("%.0f", $min_age_upd) . " days old: '$file_comscore_updates_latest'\n" ;
+
+ if ($min_age_upd == 999999)
+ {
+ print "No valid update file found. Nothing to update." ;
+ return ;
+ }
+
+ #if ($age_all > $min_age_upd)
+ #{
+ # print "File with master data more recent than latest update csv from comScore. Nothing to update." ;
+ # return ;
+ #}
+
+ my $updates_found = $false ;
+
+ open CSV, '<', $file_comscore_updates_latest ;
+ binmode CSV ;
+ while ($line = <CSV>)
+ {
+ chomp $line ;
+ $line =~ s/\r//g ;
+ $line = &GetNumberOnly ($line) ;
+
+ if ($line =~ /Jan-\d\d\d\d.*?Feb-\d\d\d\d/) # e.g. 'Location,Location,Jan-2010,Feb-2010,Mar-2010,Apr-2010,...'
+ {
+ if ($layout_csv == $layout_csv_properties)
+ { ($dummy1,$dummy2,$dummy3,@months) = split (',', $line) ; } # web properties csv file
+ else
+ { ($dummy1,$dummy2,@months) = split (',', $line) ; } # uv / reach csv files
+
+ @months = &mmm_yyyy2yyyy_mm (@months) ;
+ }
+
+ if (($line =~ /^\d+,/) || ($line =~ /,,.*?Total Internet/))
+ {
+ if ($layout_csv == $layout_csv_properties)
+ {
+ ($index,$dummy,$property,@data) = split (',', $line) ;
+ $property =~ s/^\s+// ;
+ $property =~ s/\s+$// ;
+
+ $property =~ s/.*Total Internet.*/Total Internet/i ;
+ $property =~ s/.*Google.*/Google/i ;
+ $property =~ s/.*Microsoft.*/Microsoft/i ;
+ $property =~ s/.*FACEBOOK.*/Facebook/i ;
+ $property =~ s/.*Yahoo.*/Yahoo/i ;
+ $property =~ s/.*Amazon.*/Amazon/i ;
+ $property =~ s/.*Apple.*/Apple/i ;
+ $property =~ s/.*AOL.*/AOL/i ;
+ $property =~ s/.*Wikimedia.*/Wikimedia/i ;
+ $property =~ s/.*Tencent.*/Tencent/i ;
+ $property =~ s/.*Baidu.*/Baidu/i ;
+ $property =~ s/.*CBS.*/CBS/i ;
+
+ if (! $white_list {$property})
+ {
+ $not_white_listed {$property}++ ;
+ next ;
+ }
+
+ $id = $property ;
+ }
+ else
+ {
+ ($index,$region,@data) = split (',', $line) ;
+ $region =~ s/^\s+// ;
+ $region =~ s/\s+$// ;
+
+ if (! $white_list {$region})
+ {
+ $not_white_listed {$region}++ ;
+ next ;
+ }
+
+ $id = $region_codes {$region} ;
+ }
+
+ for ($m = 0 ; $m <= $#months ; $m++)
+ {
+ $yyyymm = $months [$m] ;
+ $months {$yyyymm} ++ ;
+ $yyyymm_id = "$yyyymm,$id" ;
+ $data = $data [$m] * $multiplier ;
+
+ if ($mode eq 'add')
+ {
+ if (! defined $data {$yyyymm_id})
+ {
+ $updates_found = $true ;
+ print "New data found: $yyyymm_id = $data\n" ;
+ $data {$yyyymm_id} = $data ;
+ }
+ }
+ else
+ {
+ $updates_found = $true ;
+ print "Data found: $yyyymm_id = $data\n" ;
+ $data {$yyyymm_id} = $data ;
+ }
+ }
+ }
+ }
+
+ $entities_not_white_listed = join (', ', sort keys %not_white_listed) ;
+ if ($entities_not_white_listed ne '')
+ { print "\nEntities ignored:\n$entities_not_white_listed\n\n" ; }
+
+ if (! $updates_found)
+ { print "No new updates found\n" ; }
+ else
+ { print "\nUpdates found, rewrite master file '$file_comscore_master'\n\n" ; }
+
+ return ($updates_found) ;
+}
+
+sub WriteDataAnalyticsMySQL
+{
+ print "WriteDataAnalyticsMySQL\n\n";
+
+ open OUT, '>', "$dir_out/analytics_in_comscore.csv" ;
+
+ $metric = 'unique_visitors' ;
+ foreach $yyyymm (sort keys %months)
+ {
+ # store meta data elsewhere
+ # $line = "$generated,$source,$server,$script_name,$script_version,$user,$yyyymm,$country_code,$region_code,$property,$project,$normalized,$metric,$data\n" ;
+ foreach $region_code (sort values %region_codes)
+ {
+ $country_code = '-' ;
+ $property = '-' ;
+ $project = '-' ;
+ $reach = $reach_region_code {"$yyyymm,$region_code"} ;
+ $visitors = $visitors_region_code {"$yyyymm,$region_code"} ;
+
+ if (! defined $reach) { $reach = -1 ; }
+ if (! defined $visitors) { $visitors = -1 ; }
+
+ next if $reach == -1 and $visitors == -1 ;
+
+ $line = "$yyyymm,$country_code,$region_code,$property,$project,$reach,$visitors\n" ;
+ print OUT $line ;
+ print $line ;
+ }
+
+ foreach $property (sort @properties)
+ {
+ $country_code = '-' ;
+ $region_code = '-' ;
+ $project = '-' ;
+ $reach = '-1' ;
+ $visitors = $visitors_web_property {"$yyyymm,$property"} ;
+
+ next if ! defined $visitors ;
+
+ $line = "$yyyymm,$country_code,$region_code,$property,$project,$reach,$visitors\n" ;
+ print OUT $line ;
+ # print $line ;
+ }
+ }
+}
+
+sub WriteDataAnalyticsDygraph
+{
+ print "WriteDataAnalyticsDygraph -> $dir_out\n\n";
+
+ open CSV_REGIONS_UV, '>', "$dir_out/wikilytics_in_comscore_regions_uv.csv" ;
+ open CSV_REGIONS_REACH, '>', "$dir_out/wikilytics_in_comscore_regions_reach.csv" ;
+ open CSV_PROPERTIES_UV, '>', "$dir_out/wikilytics_in_comscore_properties_uv.csv" ;
+
+ $line = "date," ;
+ foreach $region_code (sort values %region_codes)
+ {
+ next if $region_code eq 'US' ;
+ $line .= "$region_code," ;
+ }
+ $line =~ s/,$/\n/ ;
+
+ print CSV_REGIONS_UV $line ;
+ print CSV_REGIONS_REACH $line ;
+
+ $line = "date," ;
+ foreach $property (sort @properties)
+ { $line .= "$property," ; }
+ $line =~ s/,$/\n/ ;
+
+ print CSV_PROPERTIES_UV $line ;
+
+ $metric = 'unique_visitors' ;
+ foreach $yyyymm (sort keys %months)
+ {
+ # store meta data elsewhere
+ # $line = "$generated,$source,$server,$script_name,$script_version,$user,$yyyymm,$country_code,$region_code,$property,$project,$normalized,$metric,$data\n" ;
+
+ $line_regions_uv = "$yyyymm," ;
+ $line_regions_reach = "$yyyymm," ;
+ $line_properties_uv = "$yyyymm," ;
+
+ foreach $region_code (sort values %region_codes)
+ {
+ next if $region_code eq 'US' ;
+
+ $reach = $reach_region_code {"$yyyymm,$region_code"} ;
+ $visitors = $visitors_region_code {"$yyyymm,$region_code"} ;
+
+ if (! defined $reach) { $reach = -1 ; }
+ if (! defined $visitors) { $visitors = -1 ; }
+
+ $line_regions_uv .= "$visitors," ;
+ $line_regions_reach .= "$reach," ;
+ }
+
+ foreach $property (sort @properties)
+ {
+ $visitors = $visitors_web_property {"$yyyymm,$property"} ;
+
+ if (! defined $visitors) { $visitors = -1 ; }
+
+ $line_properties_uv .= "$visitors," ;
+ }
+
+ $line_regions_uv =~ s/,$/\n/ ;
+ $line_regions_reach =~ s/,$/\n/ ;
+ $line_properties_uv =~ s/,$/\n/ ;
+
+ print CSV_REGIONS_UV $line_regions_uv ;
+ print CSV_REGIONS_REACH $line_regions_reach ;
+ print CSV_PROPERTIES_UV $line_properties_uv ;
+ }
+}
+
+sub GetNumberOnly
+{
+ my $line = shift ;
+ $line =~ s/("[^\"]*")/($a=$1,$a=~s#,##g,$a)/ge ; # nested regexp: remove comma's inside double quotes
+ $line =~ s/"//g ;
+ return $line ;
+}
+
+sub mmm_yyyy2yyyy_mm
+{
+ my @months = @_ ;
+ my ($m) ;
+ # Jan -> 01, etc
+ foreach $month (@months)
+ {
+ my ($mmm,$yyyy) = split ('-', $month) ;
+ for ($m = 0 ; $m <= $#months_short ; $m++)
+ {
+ if ($mmm eq $months_short [$m])
+ {
+ $month = "$yyyy-" . sprintf ("%02d", $m+1) ;
+ last ;
+ }
+ }
+ }
+ return @months ;
+}
+
+sub Abort
+{
+ $msg = shift ;
+
+ print "\nAbort, reason: $msg\n\n" ;
+ exit ;
+}
+
+sub days_in_month
+{
+ my $year = shift ;
+ my $month = shift ;
+ my $days = $days_in_month_cached {"$year $month"} ;
+ return $days if $days > 0 ;
+
+ my $month2 = $month+1 ;
+ my $year2 = $year ;
+ if ($month2 > 12)
+ { $month2 = 1 ; $year2++ }
+
+ my $timegm1 = timegm (0,0,0,1,$month-1,$year-1900) ;
+ my $timegm2 = timegm (0,0,0,1,$month2-1,$year2-1900) ;
+ $days = ($timegm2-$timegm1) / (24*60*60) ;
+
+ $days_in_month_cached {"$year $month"} = $days ;
+ return ($days) ;
+}
+

Added: trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepLanguageNames.pl
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepLanguageNames.pl (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepLanguageNames.pl 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,162 @@
+#!/usr/bin/perl
+
+# This module prepares a csv file with language names for feed into the analytics database
+# The main work of collecting these names from different sources is done by
+# http://svn.wikimedia.org/viewvc/mediawiki/trunk/wikistats/dumps/WikiReportsLocalizations.pm
+# which is part of wikistats reporting phase and once each month updates local master csv files from all sources
+
+# The following code to merge and filter these master csv files is based on parts of the code in WikiReportsLocalizations.pm, function Localization
+# To do some day: make a completely independent script out of this code and code from WikiReportsLocalizations.pm which covers the whole production cycle
+
+# Sources for language names:
+# - php files
+# - translatewiki
+# - English Wikipedia API (interwikilinks)
+
+# Right now multilingual support for the analytics database is just a nice idea, so to speed up data feeds, just keep English and German translations
+
+ use Getopt::Std ;
+
+ $true = 1 ;
+ $false = 0 ;
+
+ $max_language_name = 50 ; # truncate if longer
+
+ $file_csv_language_names_php = "LanguageNamesViaPhp.csv" ;
+ $file_csv_language_names_wp = "LanguageNamesViaWpEnEdited.csv" ;
+ $file_csv_analytics_in = "analytics_in_language_names.csv" ;
+
+ $languages_force_case_uc = "ast|br|de|en|id|nl|wa" ; # incomplete list, based on languages supported by wikistats reporting
+ $languages_filter = "de|en" ;
+ foreach $language (split '\|', $languages_filter)
+ { $include_target_language {$language} = $true ; }
+
+ &ParseArguments ;
+ &ReadCsvFiles ;
+ &WriteCsvFile ;
+
+
+# if ($language eq "id") # swap which file takes precedence
+
+ print "\nReady\n\n" ;
+ exit ;
+
+sub ParseArguments
+{
+ my (@options, $arguments) ;
+
+ getopt ("io", \%options) ;
+
+ foreach $arg (sort keys %options)
+ { $arguments .= " -$arg " . $options {$arg} . "\n" ; }
+ print ("\nArguments\n$arguments\n") ;
+
+ $options {"i"} = "w:/# out bayes/csv_wp" ; # EZ test
+ $options {"o"} = "c:/MySQL/analytics" ; # EZ test
+
+ die ("Specify input folder for projectcounts files as: -i path") if (! defined ($options {"i"})) ;
+ die ("Specify output folder as: -o path'") if (! defined ($options {"o"})) ;
+
+ ($path_in = $options {"i"}) =~ s/[\/\\]+$// ; # remove trailing (back)slash if any
+ ($path_out = $options {"o"}) =~ s/[\/\\]+$// ; # remove trailing (back)slash if any
+
+ die "Input folder '$path_in' does not exist" if (! -d $path_in) ;
+ die "Output folder '$path_out' does not exist" if (! -d $path_out) ;
+
+ print "Input folder: $path_in\n" ;
+ print "Output folder: $path_out\n\n" ;
+
+ $file_csv_language_names_php = "$path_in/$file_csv_language_names_php" ;
+ $file_csv_language_names_wp = "$path_in/$file_csv_language_names_wp" ;
+ $file_csv_analytics_in = "$path_out/$file_csv_analytics_in" ;
+
+ die "Input file '$file_csv_language_names_php' not found" if (! -e $file_csv_language_names_php) ;
+ die "Input file '$file_csv_language_names_wp' not found" if (! -e $file_csv_language_names_wp) ;
+}
+
+sub ReadCsvFiles
+{
+ #first read definitions from php message files, then overwrite with definitions from interwiki links when available
+ # except for target language 'id' (Indonesian) where quality of php file has been deemed more reliable
+
+ open CSV_IN, "<", $file_csv_language_names_php ;
+ while ($line = <CSV_IN>)
+ {
+ chomp ($line) ;
+ ($target_language, $code, $name_unicode, $name_html) = split (',', $line) ;
+
+ next if ! $include_target_language {$target_language} ;
+
+ $out_languages {$target_language} {$code} = &FormatName ($target_language, $name_unicode) ; # forget about html for analytics database
+ }
+ close CSV_IN ;
+
+ open CSV_IN, "<", $file_csv_language_names_wp ;
+ while ($line = <CSV_IN>)
+ {
+ chomp ($line) ;
+ ($target_language, $code, $name_unicode, $name_html) = split (',', $line) ;
+
+ next if ! $include_target_language {$target_language} ;
+
+ next if $target_language eq 'id' and $out_languages {$target_language} {$code} ne '' ;
+
+ # $name_unicode_php = $out_languages {$target_language} {$code} ; # test only
+ # $name_unicode_wp = &FormatName ($target_language, $name_unicode) ; # test only
+ # if (($name_unicode_php ne '') && ($name_unicode_php ne $name_unicode_wp)) # test only
+ # { print "$name_unicode_php => $name_unicode_wp\n" ; } # test only
+
+ $out_languages {$target_language} {$code} = &FormatName ($target_language, $name_unicode) ; # forget about html for analytics database
+ }
+ close CSV_IN ;
+}
+
+sub FormatName
+{
+ my ($target_language, $name_unicode) = @_ ;
+
+ $name_unicode2 = $name_unicode ;
+
+ if ($target_language eq "de")
+ { $name_unicode =~ s/e?\s*\-?sprache//i ; }
+
+ if ($target_language =~ /^(?:$languages_force_case_uc)/)
+ { $name_unicode = ucfirst $name_unicode ; }
+ else
+ { $name_unicode = lc $name_unicode ; }
+
+# Test only
+# if (($target_language eq 'de') && ($name_unicode ne $name_unicode2))
+# { print "$name_unicode2 => $name_unicode\n" ; }
+
+ return ($name_unicode) ;
+}
+
+sub WriteCsvFile
+{
+ open CSV_OUT, ">", $file_csv_analytics_in || die ("File '$file_csv_analytics_in' could not be opened") ;
+ binmode CSV_OUT ; # force Unix style linebreak \012
+
+ foreach $target_language (sort keys %out_languages)
+ {
+ print "\nTarget language '$target_language'\n" ;
+ %translations = %{$out_languages {$target_language}} ;
+
+ foreach $code (sort keys %translations)
+ {
+ $language_name = $translations{$code} ;
+ $language_name2 = substr ($language_name,0,$max_language_name) ;
+
+ if ($language_name ne $language_name2)
+ { print "Language name truncated to $max_language_name chars: '$language_name' -> '$language_name2'\n" ; }
+
+ if ($language_name2 =~ /,/)
+ { $language_name2 = "\"$language_name2\"" ; }
+ # test only
+ print CSV_OUT "$target_language,$code,$language_name2\n" ;
+ }
+ }
+
+ close CSV_OUT ;
+}
+

Added: trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepPageViews.pl
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepPageViews.pl (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepPageViews.pl 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,11 @@
+#!/usr/bin/perl
+
+# This file is just a place holder (alias)
+# Page views per day/week/month/etc, per project, per language, nrmalized and not normalized are collected by
+# http://svn.wikimedia.org/viewvc/mediawiki/trunk/wikistats/dumps/WikiCountsSummarizeProjectCounts.pl
+#
+# Added May 2001:
+# For analytics database one file is written for all projects and languages combined,
+# with per month, not normalized and normalized page view counts in one row.
+# This file is written to same output folder as other csv files generated by WikiCountsSummarizeProjectCounts.pl
+# and needs to be copied to analytics folder later
\ No newline at end of file

Added: trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutput.pl
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutput.pl (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutput.pl 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,3 @@
+#!/usr/local/bin/perl
+print "2012-06-04 renamed to AnalyticsPrepWikiCountsOutputMisc.pl" ;
+exit ;

Added: trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutputCore.pl
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutputCore.pl (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutputCore.pl 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,1212 @@
+#!/usr/local/bin/perl
+
+ use lib "/home/ezachte/lib" ;
+ use EzLib ;
+ $trace_on_exit = $true ;
+ ez_lib_version (2) ;
+
+ $month_last = sprintf ("%02d", $month_last) ; # 1 -> 01
+
+ # set defaults mainly for tests on local machine
+ default_argv "-i 'W:/# Out Bayes'|-o 'W:/@ Report Card/Data'|-m 2012-04" ;
+
+ use Getopt::Std ;
+
+# $file_regions_UV = "Multi-Country Media Trend, UVs by region (July 2008 - September 2009)_27290.csv" ;
+# $file_regions_Reach = "Multi-Country Media Trend, % reach by region (July 2008 - September 2009)_10786.csv" ;
+
+ $maxpopularwikis = 25 ;
+ @projects = ('wb','wk','wn','wp','wq','ws','wv','wx','commons','*') ;
+
+ &LogArguments ;
+ &ParseArguments ;
+ &InitProjectNames ;
+ &InitReportNames ;
+ &ReadStatisticsMonthly ;
+ &WriteMonthlyData ; # filename for old report card process
+ &CopyMonthlyData ; # filename for new report card process
+ exit ;
+
+sub LogArguments
+{
+ my $arguments ;
+ getopt ("iom", \%options) ;
+ foreach $arg (sort keys %options)
+ { $arguments .= " -$arg " . $options {$arg} . "\n" ; }
+ print ("\nArguments\n$arguments\n") ;
+# &Log ("\nArguments\n$arguments\n") ;
+}
+
+sub ParseArguments
+{
+ my @options ;
+ getopt ("iom", \%options) ;
+
+ die ("Specify input folder for projectcounts files as: -i path") if (! defined ($options {"i"})) ;
+ die ("Specify output folder as: -o path'") if (! defined ($options {"o"})) ;
+ die ("Specify month as -m yyyy-mm") if (! defined ($options {"m"})) ;
+
+ $path_in = $options {"i"} ;
+ $path_out = $options {"o"} ;
+ $yyyy_mm = $options {"m"} ;
+
+ die "Input folder '$path_in' does not exist" if (! -d $path_in) ;
+ die "Output folder '$path_out' does not exist" if (! -d $path_out) ;
+ die "Specify month as -m yyyy-mm" if ($yyyy_mm !~ /^\d\d\d\d-\d\d$/) ;
+
+ $month_last = substr ($options {"m"},5,2) ;
+ $year_last = substr ($options {"m"},0,4) ;
+
+ $month_start = $month_last ;
+ $year_start = $year_last - 3 ;
+
+ $m_start = &months_since_2000_01 ($year_start, $month_start) ;
+ $m_last = &months_since_2000_01 ($year_last, $month_last) ;
+ $m_last_12 = $m_last - 12 ;
+ $m_last_1 = $m_last - 1 ;
+
+ print "Input folder: $path_in\n" ;
+ print "Output folder: $path_out\n" ;
+ print "Period " . sprintf ("%04d-%02d", $year_start, $month_start) . ' - ' . sprintf ("%04d-%02d", $year_last, $month_last) . "\n" ;
+ print "\n" ;
+
+ $file_csv_out = "$path_out/StatisticsMonthly_${year_last}_${month_last}.csv" ;
+
+ &SetComparisonPeriods ;
+}
+
+sub ReadStatisticsMonthly
+{
+ &ReadStatisticsMonthlyForProject ("wb") ;
+ &ReadStatisticsMonthlyForProject ("wk") ;
+ &ReadStatisticsMonthlyForProject ("wn") ;
+ &ReadStatisticsMonthlyForProject ("wp") ;
+ &ReadStatisticsMonthlyForProject ("wq") ;
+ &ReadStatisticsMonthlyForProject ("ws") ;
+ &ReadStatisticsMonthlyForProject ("wv") ;
+ &ReadStatisticsMonthlyForProject ("wx") ;
+
+ &ReadStatisticsPerBinariesExtensionCommons ;
+}
+
+sub ReadStatisticsMonthlyForProject
+{
+ my $project = shift;
+
+ $all_projects = "*" ;
+
+ my $file_csv_in_1 = "$path_in/csv_$project/StatisticsMonthly.csv" ;
+ my $file_csv_in_2 = "$path_in/csv_$project/StatisticsUserActivitySpread.csv" ;
+
+ if (! -e $file_csv_in_1)
+ { &Abort ("Input file '$file_csv_in_1' not found") ; }
+ if (! -e $file_csv_in_2)
+ { &Abort ("Input file '$file_csv_in_2' not found") ; }
+
+ print "Read '$file_csv_in_1'\n" ;
+ open CSV_IN, '<', $file_csv_in_1 ;
+
+ undef %lines ;
+ while ($line = <CSV_IN>)
+ {
+ ($language,$date,$counts) = split (',', $line, 3) ;
+ next if $language =~ /^zz/ ;
+ next if $language eq 'commons' and $project ne 'wx' ;
+ next if $language eq 'sr' and $project eq 'wn' ; # ignore insane bot spam on
+
+ ($month,$day,$year) = split ('\/', $date) ;
+ my $m = &months_since_2000_01 ($year,$month) ;
+ next if $m < $m_start ;
+
+ $lines {$language}{$m} = $line ;
+ $languages {$language}++ ;
+ }
+
+ foreach $language (sort keys %languages)
+ {
+ for ($m = $m_start + 1 ; $m <= $m_last ; $m++)
+ {
+ if ($lines {$language}{$m} eq '')
+ { $lines {$language}{$m} = $lines {$language}{$m -1} ; }
+ }
+
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ {
+ $line = $lines {$language}{$m} ;
+ chomp $line ;
+ ($language,$date,$counts) = split (',', $line, 3) ;
+ @fields = split (',', $counts) ;
+
+ if ($project eq "wp")
+ {
+ foreach $f (1,4,6,11) # new editors, articles, new articles, edits
+ {
+ $values {"$f,$m"} {"$project,$language"} = $fields [$f] ;
+
+ $totals {"$f,$m"} += $fields [$f] ;
+
+ $totals_project {"$f,$m"} {$project} += $fields [$f] ;
+
+ $totals_project {"$f,$m"} {$all_projects} += $fields [$f] ;
+
+ # print "TOTALS $f $m = . " . $totals {"$f,$m"} . "\n" ;
+ }
+ }
+ else
+ {
+ foreach $f (1,4)
+ {
+ if ($f <= 3)
+ {
+ $values {"$f,$m"} {"$project,$language"} = $fields [$f] ;
+ $totals {"$f,$m"} += $fields [$f] ;
+ }
+
+
+ # ignore editor count on commons for totals, most editors are already counted for other project
+ # (even for several projects, to be tuned after centralauth dump is available)
+ # count for all_projects only Wikipedia articles
+ if (($f <= 3) && ($language ne 'commons')) # 0 = Contributors, 1 = New Wikimedians, 2 = Active Editors (5+ edits), 3 = Very Active Editors (100+ edits),
+ { $totals_project {"$f,$m"} {$all_projects} += $fields [$f] ; }
+
+ if ($language eq 'commons')
+ { $totals_project {"$f,$m"} {'commons'} += $fields [$f] ; }
+ else
+ { $totals_project {"$f,$m"} {$project} += $fields [$f] ; }
+
+ # print "TOTALS $f $m = . " . $totals {"$f,$m"} . "\n" ;
+ }
+ foreach $f (6,11)
+ {
+ $totals_project {"$f,$m"} {$all_projects} += $fields [$f] ;
+ if ($language eq 'commons')
+ { $totals_project {"$f,$m"} {'commons'} += $fields [$f] ; }
+ else
+ { $totals_project {"$f,$m"} {$project} += $fields [$f] ; }
+ # print "TOTALS $f $m = . " . $totals {"$f,$m"} . "\n" ;
+ }
+
+ }
+ }
+ }
+ close CSV_IN ;
+
+ # now read (very) active editors from newer more accurate file (split data for reg users and bots, unlike StatisticsMonthly.csv)
+ # but use f = column count in StatisticsMonthly.csv
+
+ print "Read '$file_csv_in_2'\n" ;
+ open CSV_IN, '<', $file_csv_in_2 ;
+
+ undef %lines ;
+ while ($line = <CSV_IN>)
+ {
+ chomp $line ;
+ ($language,$date,$reguser_bot,$group,$counts) = split (',', $line, 5) ;
+ next if $language =~ /^zz/ ;
+
+ next if $language eq 'commons' and $project ne 'wx' ; # commons also in wikipedia csv files (bug, hard to cleanup, just skip)
+ # next if $language eq 'commons' ; # ignore editor count on commons alltogether, most are already counted for other project
+ # (even for several projects, to be tuned after centralauth dump is available)
+
+ if ($reguser_bot ne "R") { next ; } # R: reg user, B: bot
+ if ($group ne "A") { next ; } # A: articles, T: talk pages, O: other namespaces
+
+ ($month,$day,$year) = split ('\/', $date) ;
+ my $m = &months_since_2000_01 ($year,$month) ;
+ next if $m < $m_start ;
+
+ $lines {$language}{$m} = $line ;
+ $languages {$language}++ ;
+ }
+
+ foreach $language (sort keys %languages)
+ {
+ for ($m = $m_start+1 ; $m <= $m_last ; $m++)
+ {
+ if ($lines {$language}{$m} eq '')
+ { $lines {$language}{$m} = $lines {$language}{$m -1} ; }
+ }
+
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ {
+ $line = $lines {$language}{$m} ;
+ chomp $line ;
+ ($language,$date,$reguser_bot,$group,$counts) = split (',', $line, 5) ;
+ @fields = split (',', $counts) ;
+
+ foreach $f (2,3) # editors_gt_5, editors_gt_100
+ {
+ # count user with over x edits
+ # threshold starting with a 3 are 10xSQRT(10), 100xSQRT(10), 1000xSQRT(10), etc
+ # thresholds = 1,3,5,10,25,32,50,100,etc
+ if ($f == 2) { $f2 = 2 ; }
+ if ($f == 3) { $f2 = 7 ; }
+
+ $values {"$f,$m"} {"$project,$language"} = $fields [$f2] ;
+ $totals {"$f,$m"} += $fields [$f2] ;
+
+ # ignore editor count on commons for totals, most editors are already counted for other project
+ # (even for several projects, to be tuned after centralauth dump is available)
+ if (($f <= 3) && ($language ne 'commons')) # 0 = Contributors, 1 = New Wikimedians, 2 = Active Editors (5+ edits), 3 = Very Active Editors (100+ edits),
+ { $totals_project {"$f,$m"} {$all_projects} += $fields [$f2] ; }
+
+ if ($language eq 'commons')
+ { $totals_project {"$f,$m"} {'commons'} += $fields [$f2] ; }
+ else
+ { $totals_project {"$f,$m"} {$project} += $fields [$f2] ; }
+ }
+ }
+ }
+ close CSV_IN ;
+}
+
+sub ReadStatisticsPerBinariesExtensionCommons
+{
+ my $file_csv_in = "$path_in/csv_wx/StatisticsPerBinariesExtension.csv" ;
+ my $mmax = -1 ;
+
+ if (! -e $file_csv_in)
+ { &Abort ("Input file '$file_csv_in' not found") ; }
+
+ print "Read '$file_csv_in'\n" ;
+ open CSV_IN, '<', $file_csv_in ;
+ while ($line = <CSV_IN>)
+ {
+ chomp $line ;
+ ($language,$date,$counts) = split (',', $line, 3) ;
+
+ if ($language ne "commons") { next ; }
+ if ($date eq "00/0000")
+ {
+ @fields = split (',', $counts) ;
+ $field_ndx = 0 ;
+ foreach $field (@fields)
+ {
+ $ext_cnt {-1}{$field_ndx} = $field ;
+ # print "EXT_CNT $field_ndx : $field\n" ;
+ $field_ndx ++ ;
+ }
+ next ;
+ }
+
+ ($month,$year) = split ('\/', $date) ;
+ my $m = &months_since_2000_01 ($year,$month) ;
+ next if $m < $m_start ;
+
+ if ($m > $mmax)
+ { $mmax = $m ; }
+
+ @fields = split (',', $counts) ;
+ $field_ndx = 0 ;
+ foreach $field (@fields)
+ {
+ $ext_cnt {$m}{$field_ndx} = $field ;
+ $ext_tot {$m} += $field ;
+ $field_ndx ++ ;
+ }
+ }
+ close CSV_IN ;
+
+ %ext_cnt_mmax = %{$ext_cnt {$mmax}} ;
+ @ext_cnt_mmax = (sort {$ext_cnt_mmax {$b} <=> $ext_cnt_mmax {$a}} keys %ext_cnt_mmax) ;
+
+ $extcnt = 0 ;
+ foreach $extndx (@ext_cnt_mmax)
+ {
+ # print "$extndx < ${ext_cnt {-1}{$extndx}} > : ${ext_cnt_mmax {$extndx}}\n" ;
+ push @extndxs, $extndx ;
+ if ($extcnt++ >= 9) { last ; }
+ }
+}
+
+sub ReadMediaTrends
+{
+# open FILE_UV, '<', $file_regions_UV ;
+# close FILE-UV ;
+
+# open FILE_REACH, '<', $file_regions_Reach ;
+# close FILE_REACH ;
+}
+
+sub WriteMonthlyData
+{
+ print "Write file '$file_csv_out'\n" ;
+ open CSV_OUT, '>', $file_csv_out ;
+ $output = "" ;
+
+ foreach $f (1,2,3,4,6,11) # new editors, editors_gt_5, editors_gt_100, articles, new articles, edits
+ {
+
+ $output .= "\n,${out_report_descriptions [$f]} - Absolute - Per Wiki\n" ;
+ $output .= "$csv_recent_months,%inc year, %inc month\n" ;
+
+ $line = ",Total," ;
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ {
+ if ($f <= 3) # new editors, editors_gt_5, editors_gt_100,
+ { $line .= $totals_project {"$f,$m"} {$all_projects} . "," ; }
+ else
+ { $line .= $totals {"$f,$m"} . "," ; }
+ }
+
+ # growth in one year
+ if ($totals {"$f,$m_last_12"} != 0)
+ { $line .= sprintf ("%.1f", 100 * ($totals {"$f,$m_last"} / $totals {"$f,$m_last_12"}) - 100). "%," ; }
+ else
+ { $line .= "n.a.," ; }
+
+ # growth in one month
+ if ($totals {"$f,$m_last_1"} != 0)
+ { $line .= sprintf ("%.1f", 100 * ($totals {"$f,$m_last"} / $totals {"$f,$m_last_1"}) - 100). "%," ; }
+ else
+ { $line .= "n.a.," ; }
+
+ $line =~ s/,$// ;
+ $output .= "$line\n" ;
+
+ # sort by absolute amount for last month
+ %values_f_12 = %{$values {"$f,$m_last"}} ;
+ $index = 1 ;
+ foreach $key (sort {$values_f_12 {$b} <=> $values_f_12 {$a}} keys %values_f_12)
+ {
+ ($project,$language) = split (",", $key) ;
+ $language_name = $out_languages {$language} ;
+ if (($project ne "wp") && ($project ne "wx"))
+ { $line = "$index,$language_name " . &GetProjectName ($project) . "," ; }
+ else
+ { $line = "$index,$language_name," ; }
+
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ { $line .= $values {"$f,$m"} {$key} . "," ; }
+
+ if ($values {"$f,$m_last_12"} {$key} != 0)
+ { $line .= sprintf ("%.1f", 100 * ($values {"$f,$m_last"} {$key} / $values {"$f,$m_last_12"} {$key}) - 100). "%," ; }
+ else
+ { $line .= "n.a.," ; }
+
+ if ($values {"$f,$m_last_1"} {$key} != 0)
+ { $line .= sprintf ("%.1f", 100 * ($values {"$f,$m_last"} {$key} / $values {"$f,$m_last_1"} {$key}) - 100). "%," ; }
+ else
+ { $line .= "n.a.," ; }
+
+ $line =~ s/,$// ;
+ $output .= "$line\n" ;
+
+ if ($index++ >= 25) { last ; }
+ }
+
+ $output .= "\n,${out_report_descriptions [$f]} - Absolute - Per Project\n" ;
+ if ($f <= 3) # 0 = Contributors, 1 = New Wikimedians, 2 = Active Editors (5+ edits), 3 = Very Active Editors (100+ edits),
+ { $output .= ",Note: All projects does not include Commons\n" ; }
+ $output .= "$csv_recent_months,%inc year, %inc month\n" ;
+
+ foreach $project (sort {$totals_project {"$f,$m_last"} {$b} <=> $totals_project {"$f,$m_last"} {$a}} @projects)
+ {
+# next if $project eq 'commons' and ($f ==2 or $f == 3) ; # (very) active editors no longer counted for commons
+
+ if ($project eq 'commons')
+ { $line = ",Commons," ; }
+ else
+ { $line = "," . &GetProjectName ($project) . "," ; }
+
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ { $line .= $totals_project {"$f,$m"} {$project} . "," ; }
+
+ if ($totals_project {"$f,$m_last_12"} {$project} != 0)
+ { $line .= sprintf ("%.1f", 100 * ($totals_project {"$f,$m_last"} {$project} / $totals_project {"$f,$m_last_12"} {$project}) - 100). "%," ; }
+ else
+ { $line .= "n.a.," ; }
+
+ if ($totals_project {"$f,$m_last_1"} {$project} != 0)
+ { $line .= sprintf ("%.1f", 100 * ($totals_project {"$f,$m_last"} {$project} / $totals_project {"$f,$m_last_1"} {$project}) - 100). "%," ; }
+ else
+ { $line .= "n.a.," ; }
+
+ $line =~ s/,$// ;
+ $output .= "$line\n" ;
+ }
+
+ $output .= "\n,${out_report_descriptions [$f]} - Indexed - Per Wiki\n" ;
+ $output .= "$csv_recent_months\n" ;
+
+ # sort by absolute amount for last month
+ $index = 1 ;
+ foreach $key (sort {$values_f_12 {$b} <=> $values_f_12 {$a}} keys %values_f_12)
+ {
+ # print "$index $f: $key -> ${values_f_12 {$key}}\n" ;
+
+ ($project,$language) = split (",", $key) ;
+if ($f == 2)
+{ print "project $project $language\n" ; }
+ $language_name = $out_languages {$language} ;
+ if (($project ne "wp") && ($project ne "wx"))
+ { $line = "$index,$language_name " . &GetProjectName ($project) . "," ; }
+ else
+ { $line = "$index,$language_name," ; }
+
+ # $value_100 = $values {"$f,$m_last_12"} {$key} ;
+ $value_100 = $values {"$f,$m_start"} {$key} ;
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ {
+ if ($value_100 != 0)
+ { $line .= sprintf ("%.1f", 100 * ($values {"$f,$m"} {$key} / $value_100)) . "," ; }
+ else
+ { $line .= "," ; }
+ }
+ $line =~ s/,$// ;
+ $output .= "$line\n" ;
+
+ # put totals last in chart to show line on top of others
+ if ($index == 9)
+ {
+ $line = ",Total," ;
+ $total_100 = $totals {"$f,$m_last_12"} ;
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ {
+ if ($total_100 != 0)
+ { $line .= sprintf ("%.1f", 100 * ($totals {"$f,$m"} / $total_100)) . "," ; }
+ else
+ { $line .= "," ; }
+ }
+ $line .= ",(sorted here to make it top-most line out of 10 in Excel)" ;
+ $output .= "$line\n" ;
+ }
+
+ if ($index++ >= 25) { last ; }
+ }
+
+ $output .= "\n,${out_report_descriptions [$f]} - Indexed - Per Project\n" ;
+ $output .= "$csv_recent_months,%inc year, %inc month\n" ;
+
+ foreach $project (sort {$totals_project {"$f,$m_last"} {$b} <=> $totals_project {"$f,$m_last"} {$a}} @projects)
+ {
+# next if $project eq 'commons' and ($f ==2 or $f == 3) ; # (very) active editors no longer counted for commons
+
+ if ($project eq 'commons')
+ { $line = ",Commons," ; }
+ else
+ { $line = "," . &GetProjectName ($project) . "," ; }
+
+ # $value_100 = $totals_project {"$f,$m_last_12"} {$project} ;
+ $value_100 = $totals_project {"$f,$m_start"} {$project} ;
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ {
+ if ($value_100 != 0)
+ { $line .= sprintf ("%.1f", 100 * ($totals_project {"$f,$m"} {$project} / $value_100)) . "," ; }
+ else
+ { $line .= "," ; }
+ }
+ $line =~ s/,$// ;
+ $output .= "$line\n" ;
+ }
+ $output .= "\n," . '=' x 150 . "\n" ;
+ }
+
+ print CSV_OUT $output ;
+
+ $output = "\n,Binaries per month - Absolute\n" ;
+ $output .= "$csv_recent_months,%inc year, %inc month\n" ;
+ $output .= "\n$csv_recent_months,%inc year,%inc month\n" ;
+
+ $line = ",Total," ;
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ { $line .= $ext_tot {$m} . "," ; }
+
+ if ($ext_tot {$m_last_12} != 0)
+ { $line .= sprintf ("%.1f", 100 * ($ext_tot {$m_last} / $ext_tot {$m_last_12}) - 100). "%," ; }
+ else
+ { $line .= "n.a.," ; }
+
+ if ($ext_tot {$m_last_1} != 0)
+ { $line .= sprintf ("%.1f", 100 * ($ext_tot {$m_last} / $ext_tot {$m_last_1}) - 100). "%," ; }
+ else
+ { $line .= "n.a.," ; }
+
+ $line =~ s/,$// ;
+ $output .= "$line\n" ;
+
+ $index = 0 ;
+ # feed the 10 extensions with most pages, largest one last (comes on top in Excel chart)
+ for ($e = $#extndxs - 9 ; $e <= $#extndxs ; $e++)
+ {
+ $index++ ;
+
+ if ($e < 0)
+ {
+ $line = "$index,xxx," ;
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ { $line .= "," ; }
+ }
+ else
+ {
+ $extndx = $extndxs [$e] ;
+ $line = "$index,${ext_cnt {-1}{$extndx}}," ;
+
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ { $line .= $ext_cnt {$m}{$extndx} . "," ; }
+
+ if ($ext_cnt {$m_last_12}{$extndx} != 0)
+ { $line .= sprintf ("%.1f", 100 * ($ext_cnt {$m_last}{$extndx} / $ext_cnt {$m_last_12}{$extndx}) - 100). "%," ; }
+ else
+ { $line .= "n.a.," ; }
+
+ if ($ext_cnt {$m_last_1}{$extndx} != 0)
+ { $line .= sprintf ("%.1f", 100 * ($ext_cnt {$m_last}{$extndx} / $ext_cnt {$m_last_1}{$extndx}) - 100). "%," ; }
+ else
+ { $line .= "n.a.," ; }
+ }
+
+ $line =~ s/,$// ;
+ $output .= "$line\n" ;
+ }
+
+ print CSV_OUT $output ;
+
+ $output = "\n,Binaries per month - Indexed\n" ;
+ $output .= "$csv_recent_months\n" ;
+
+ $index = 0 ;
+ # feed the 10 extensions with most pages, largest one last (comes on top in Excel chart)
+ for ($e = $#extndxs - 9 ; $e <= $#extndxs ; $e++)
+ {
+ $index++ ;
+
+ if ($e < 0)
+ {
+ $line = "$index,xxx," ;
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ { $line .= "," ; }
+ }
+ else
+ {
+ $extndx = $extndxs [$e] ;
+ $line = "$index,${ext_cnt {-1}{$extndx}}," ;
+ $ext_cnt_m0 = $ext_cnt {$m_last-12}{$extndx} ;
+ # $ext_cnt_m0 = $ext_cnt {$m_start}{$extndx} ;
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ {
+ if ($ext_cnt_m0 > 0)
+ { $line .= sprintf ("%.1f", 100 * ($ext_cnt {$m}{$extndx} / $ext_cnt_m0)). "," ; }
+ else
+ { $line .= "," ; }
+ }
+ }
+
+ $line =~ s/,$// ;
+ $output .= "$line\n" ;
+ }
+ print CSV_OUT $output ;
+ close CSV_OUT ;
+
+ print "\nOutput written to $file_csv_out\n\n" ;
+}
+
+# duplicate file, keep name for old report card process (for a while) copy to file for new report card process
+sub CopyMonthlyData
+{
+ $file_csv_out2 = "$path_out/wikilytics_in_wikistats_core_metrics.csv" ;
+ print "Write copy of $file_csv_out to $file_csv_out2\n" ;
+
+ open CSV_IN, '<', $file_csv_out ;
+ open CSV_OUT, '>', $file_csv_out2 ;
+ while ($line = <CSV_IN>)
+ { print CSV_OUT $line ; }
+ close CSV_IN ;
+ close CSV_OUT ;
+}
+
+sub SetComparisonPeriods
+{
+ my @months = qw(Xxx Jan Feb Mar Apr May Jun Jul Aug Sept Oct Nov Dec) ;
+
+ my ($file_year_month_last, $year_month_last, $year_month_last_minus_12, $year_month_last_minus_1) ;
+
+ $year_month_last = sprintf ("%04d/%02d",$year_last, $month_last) ; # for filenames
+ $file_year_month_last = sprintf ("%04d_%02d",$year_last, $month_last) ; # for filenames
+ $year_month_last_minus_12 = sprintf ("%04d/%02d",$year_last - 1,$month_last) ;
+ $year_month_last_minus_1 = $month_last > 1 ? sprintf ("%04d/%02d",$year_last,$month_last-1): sprintf ("%04d/%02d",$year_last - 1 ,12) ;
+
+ print "\nWrite trend data up till $year_month_last\n\n" ;
+ print "Compare with previous month: $year_month_last_minus_1, previous year: $year_month_last_minus_12\n\n" ;
+
+ $csv_recent_months = ",project," ;
+ $year = $year_start ;
+ $month = $month_start ;
+ for ($m = $m_start ; $m <= $m_last ; $m++)
+ {
+ $recent_months [$m] = sprintf ("%04d/%02d", $year, $month) ;
+ $csv_recent_months .= sprintf ("%02d/%04d", $month, $year) . "," ;
+ ($year,$month) = $month < 12 ? ($year,$month+1) : ($year+1,1) ;
+ }
+ $csv_recent_months =~ s/,$// ;
+}
+
+#sub WriteCsvFilesPerPeriod
+#{
+# foreach $period (sort keys %totals)
+# {
+# &LogT ("\nWrite totals per $period: ") ;
+# $desc = $descriptions {$period} ;
+
+# foreach $project (sort keys %{$totals {$period}})
+# {
+# &Log ("$project ") ;
+
+# $dir_out = "$path_out/csv_$project" ;
+# if (! -d $dir_out)
+# { mkdir $dir_out, 0777 ; }
+
+# $file_out = "$dir_out/$desc.csv" ;
+
+# open CSV, ">", $file_out ;
+# foreach $key (sort {$a cmp $b} keys %{$totals {$period}{$project}})
+# {
+# ($language,$yearmonth) = split (",", $key) ;
+# # print "PERIOD $period PROJECT $project KEY $key\n" ;
+# if ($period eq "month")
+# { print CSV "$language," . $date_high {"$yearmonth"} . "," . $totals{$period}{$project}{$key} . "\n" ; }
+# else
+# { print CSV "$key," . $totals{$period}{$project}{$key} . "\n" ; }
+# }
+# close CSV ;
+# }
+# }
+#}
+
+#sub WriteCsvHtmlFilesPopularWikis
+#{
+# @totals_lastmonth = sort {$totals_lastmonth {$b} <=> $totals_lastmonth {$a}} keys %totals_lastmonth ;
+
+# $dir_out = "$path_out/csv_wp" ;
+# $file_out = "$dir_out/PageViewsPerMonthPopularWikis_$file_year_month_last.csv" ;
+
+## extend with normalized counts
+## see manually created PageViewsPerMonthTop25PlusNormalizedTo100.csv
+
+# open CSV, ">", $file_out ;
+# print CSV $csv_recent_months ;
+
+# # write per popular language+wiki 13 months of page view totals
+# $lines = 0 ;
+# foreach $line (@totals_lastmonth)
+# {
+# if (++$lines > $maxpopularwikis) { last ; }
+
+# ($project, $language) = split (',', $line) ;
+# $largest_projects {"$project-$language"} ++ ;
+
+# $language_name = $out_languages {$language} ;
+
+# if (($project ne "wp") && ($project ne "wx"))
+# { print CSV "$language_name " . &GetProjectName ($project) . "," ; }
+# else
+# { print CSV "$language_name," ; }
+
+## %test = %{$totals {"month"} {"wp"} };
+## %test2 = @recent_months ;
+# for ($m = 0 ; $m <= 12 ; $m++)
+# { print CSV $totals {"month"} {$project} {"$language,${recent_months [$m]}"} . "," ; }
+# print CSV "\n" ;
+# }
+
+# print CSV "\n$csv_recent_months" ;
+
+# # write per popular language+wiki 13 months of page view totals, normalized to first month = 100
+# $lines = 0 ;
+# foreach $line (@totals_lastmonth)
+# {
+# if (++$lines > $maxpopularwikis) { last ; }
+
+# ($project, $language) = split (',', $line) ;
+# $language_name = $out_languages {$language} ;
+
+# if (($project ne "wp") && ($project ne "wx"))
+# { print CSV "$language_name " . &GetProjectName ($project) . "," ; }
+# else
+# { print CSV "$language_name," ; }
+
+# $recent_month_0 = $totals {"month"} {$project} {"$language,${recent_months [ 0]}"} ;
+# for ($m = 0 ; $m <= 12 ; $m++)
+# {
+# if ($recent_month_0 > 0)
+# { print CSV sprintf ("%.2f", 100 * $totals {"month"} {$project} {"$language,${recent_months [$m]}"} / $recent_month_0) . "," ; }
+# else
+# { print CSV "," ; }
+# }
+
+# print CSV "\n" ;
+# }
+# close CSV ;
+
+# # write ready made table rows for report card: page views top 25 movers shakers
+# foreach $key (keys %largest_projects)
+# {
+# ($project,$language) = split ('-', $key) ;
+
+# $total_lastmonth = $totals {"month"} {$project} {"$language,$month_last"} ;
+# $total_prevmonth = $totals {"month"} {$project} {"$language,$year_month_last_minus_1"} ;
+# $total_prevyear = $totals {"month"} {$project} {"$language,$year_month_last_minus_12"} ;
+
+# $perc_month = "no data" ;
+# $perc_year = "no data" ;
+
+# if ($total_prevyear > 0)
+# { $perc_year = sprintf ("%.1f", 100 * $total_lastmonth/$total_prevyear - 100) ; }
+# if ($total_prevyear > 0)
+# { $perc_month = sprintf ("%.1f", 100 * $total_lastmonth/$total_prevmonth - 100) ; }
+
+# $line = "$project-$language: $total_prevyear=>$total_lastmonth=$perc_year%, $total_prevmonth=>$total_lastmonth=$perc_month%" ;
+
+# $total_lastmonth = sprintf ("%.0f", $total_lastmonth / 1000000) ;
+
+# $project_name = &GetProjectName ($project) ;
+# $language_name = $out_languages {$language} ;
+
+# $col1 = "<td class=detail-left>$language_name $project_name</td>\n" ;
+# $col2 = "<td class=detail-blue>$total_lastmonth</td>\n" ;
+# $col3 = "<td class=detail-blue>$perc_month%</td>\n" ;
+# $col4 = "<td class=detail-blue>$perc_year%</td>\n" ;
+# $html = "<tr>\n$col1$col2$col3$col4</tr>\n" ;
+
+# $growth_figures_text {"$perc_month-$project-$language"} = $line ;
+# $growth_figures_html {"$perc_month-$project-$language"} = $html ;
+# }
+
+# $file_html = "$dir_out/PageViewsMoversShakersPopularWikis_$file_year_month_last.html" ;
+
+# open HTML, ">", $file_html ;
+# foreach $key (sort {$b <=> $a} keys %growth_figures_text)
+# {
+# print "$key: ". $growth_figures_text {$key} . "\n" ;
+# print HTML $growth_figures_html {$key} ;
+# }
+# close HTML ;
+#}
+
+
+sub GetProjectName
+{
+ my $project =shift ;
+
+ if ($project eq "wp") { $project_name = "Wikipedia"; }
+ elsif ($project eq "wb") { $project_name = "Wikibooks"; }
+ elsif ($project eq "wk") { $project_name = "Wiktionary"; }
+ elsif ($project eq "wx") { $project_name = "Other Wikis"; }
+ elsif ($project eq "wn") { $project_name = "Wikinews"; }
+ elsif ($project eq "wq") { $project_name = "Wikiquote"; }
+ elsif ($project eq "ws") { $project_name = "Wikisource"; }
+ elsif ($project eq "wv") { $project_name = "Wikiversity"; }
+ elsif ($project eq "*") { $project_name = "All projects"; }
+
+ return ($project_name) ;
+}
+
+sub MonthsSinceYearAgo
+{
+ my $year = shift ;
+ my $month = shift ;
+ return 12 - (($year_last - $year) * 12 + $month_last - $month) ;
+}
+
+sub MonthsSinceFirstMonthToShow
+{
+ my $year = shift ;
+ my $month = shift ;
+ return ($year - 2008) * 12 + ($month - 1) ;
+}
+
+# code year,month as monthes since january 2000 (1 byte)
+sub months_since_2000_01
+{
+ my $year = shift ;
+ my $month = shift ;
+ my $m = ($year - 2000) * 12 + $month ;
+ return $m ;
+}
+
+#sub Log
+#{
+# $msg = shift ;
+# print $msg ;
+# print LOG $msg ;
+#}
+
+#sub LogT
+#{
+# $msg = shift ;
+# my ($ss,$mm,$hh) = (localtime (time))[0,1,2] ;
+# my $time = sprintf ("%02d:%02d:%02d ", $hh, $mm, $ss) ;
+# $msg =~ s/^(\n*)/$1$time/s ;
+# &Log ($msg) ;
+#}
+
+sub MmSs
+{
+ my ($ss,$mm,$hh) = (localtime (time))[0,1,2] ;
+ return (sprintf ("%02d:%02d:%02d ", $hh, $mm, $ss)) ;
+}
+
+sub Abort
+{
+ my $msg = shift ;
+ print "$msg\nExecution aborted." ;
+ # to do: log also to file
+ exit ;
+}
+
+sub InitProjectNames
+{
+ # copied from WikiReports.pl
+
+ %wikipedias = (
+# mediawiki=>"http://wikimediafoundation.org Wikimedia",
+ nostalgia=>"http://nostalgia.wikipedia.org Nostalgia",
+ sources=>"http://wikisource.org Old&nbsp;Wikisource",
+ meta=>"http://meta.wikimedia.org Meta-Wiki",
+ beta=>"http://beta.wikiversity.org Beta",
+ species=>"http://species.wikipedia.org WikiSpecies",
+ commons=>"http://commons.wikimedia.org Commons",
+ foundation=>"http://wikimediafoundation.org Wikimedia&nbsp;Foundation",
+ sep11=>"http://sep11.wikipedia.org In&nbsp;Memoriam",
+ nlwikimedia=>"http://nl.wikimedia.org Wikimedia&nbsp;Nederland",
+ plwikimedia=>"http://pl.wikimedia.org Wikimedia&nbsp;Polska",
+ mediawiki=>"http://www.mediawiki.org MediaWiki",
+ dewikiversity=>"http://de.wikiversity.org Wikiversit&auml;t",
+ frwikiversity=>"http://fr.wikiversity.org Wikiversit&auml;t",
+ wikimania2005=>"http://wikimania2005.wikimedia.org Wikimania 2005",
+ wikimania2006=>"http://wikimania2006.wikimedia.org Wikimania 2006",
+ aa=>"http://aa.wikipedia.org Afar",
+ ab=>"http://ab.wikipedia.org Abkhazian",
+ af=>"http://af.wikipedia.org Afrikaans",
+ ak=>"http://ak.wikipedia.org Akan", # was Akana
+ als=>"http://als.wikipedia.org Alemannic", # was Elsatian
+ am=>"http://am.wikipedia.org Amharic",
+ an=>"http://an.wikipedia.org Aragonese",
+ ang=>"http://ang.wikipedia.org Anglo-Saxon",
+ ar=>"http://ar.wikipedia.org Arabic",
+ arc=>"http://arc.wikipedia.org Aramaic",
+ as=>"http://as.wikipedia.org Assamese",
+ ast=>"http://ast.wikipedia.org Asturian",
+ av=>"http://av.wikipedia.org Avar", # was Avienan
+ ay=>"http://ay.wikipedia.org Aymara",
+ az=>"http://az.wikipedia.org Azeri", # was Azerbaijani
+ ba=>"http://ba.wikipedia.org Bashkir",
+ bar=>"http://bar.wikipedia.org Bavarian",
+ bat_smg=>"http://bat-smg.wikipedia.org Samogitian",
+ "bat-smg"=>"http://bat-smg.wikipedia.org Samogitian",
+ bcl=>"http://bcl.wikipedia.org Central Bicolano",
+ be=>"http://be.wikipedia.org Belarusian",
+ "be-x-old"=>"http://be.wikipedia.org Belarusian (Tarashkevitsa)",
+ be_x_old=>"http://be.wikipedia.org Belarusian (Tarashkevitsa)",
+ bg=>"http://bg.wikipedia.org Bulgarian",
+ bh=>"http://bh.wikipedia.org Bihari",
+ bi=>"http://bi.wikipedia.org Bislama",
+ bm=>"http://bm.wikipedia.org Bambara",
+ bn=>"http://bn.wikipedia.org Bengali",
+ bo=>"http://bo.wikipedia.org Tibetan",
+ bpy=>"http://bpy.wikipedia.org Bishnupriya Manipuri",
+ br=>"http://br.wikipedia.org Breton",
+ bs=>"http://bs.wikipedia.org Bosnian",
+ bug=>"http://bug.wikipedia.org Buginese",
+ bxr=>"http://bxr.wikipedia.org Buryat",
+ ca=>"http://ca.wikipedia.org Catalan",
+ cbk_zam=>"http://cbk-zam.wikipedia.org Chavacano",
+ "cbk-zam"=>"http://cbk-zam.wikipedia.org Chavacano",
+ cdo=>"http://cdo.wikipedia.org Min Dong",
+ ce=>"http://ce.wikipedia.org Chechen",
+ ceb=>"http://ceb.wikipedia.org Cebuano",
+ ch=>"http://ch.wikipedia.org Chamorro", # was Chamoru
+ cho=>"http://cho.wikipedia.org Choctaw", # was Chotaw
+ chr=>"http://chr.wikipedia.org Cherokee",
+ chy=>"http://chy.wikipedia.org Cheyenne", # was Sets&ecirc;hest&acirc;hese
+ co=>"http://co.wikipedia.org Corsican",
+ cr=>"http://cr.wikipedia.org Cree",
+ crh=>"http://crh.wikipedia.org Crimean Tatar",
+ cs=>"http://cs.wikipedia.org Czech",
+ csb=>"http://csb.wikipedia.org Cashubian", # was Kashubian
+ cu=>"http://cv.wikipedia.org Old Church Slavonic",
+ cv=>"http://cv.wikipedia.org Chuvash", # was Cavas
+ cy=>"http://cy.wikipedia.org Welsh",
+ da=>"http://da.wikipedia.org Danish",
+ de=>"http://de.wikipedia.org German",
+ diq=>"http://diq.wikipedia.org Zazaki",
+ dk=>"http://dk.wikipedia.org Danish",
+ dsb=>"http://dsb.wikipedia.org Lower Sorbian",
+ dv=>"http://dv.wikipedia.org Divehi",
+ dz=>"http://dz.wikipedia.org Dzongkha",
+ ee=>"http://ee.wikipedia.org Ewe",
+ el=>"http://el.wikipedia.org Greek",
+ eml=>"http://eml.wikipedia.org Emilian-Romagnol",
+ en=>"http://en.wikipedia.org English",
+ eo=>"http://eo.wikipedia.org Esperanto",
+ es=>"http://es.wikipedia.org Spanish",
+ et=>"http://et.wikipedia.org Estonian",
+ eu=>"http://eu.wikipedia.org Basque",
+ ext=>"http://ext.wikipedia.org Extremaduran",
+ fa=>"http://fa.wikipedia.org Persian",
+ ff=>"http://ff.wikipedia.org Fulfulde",
+ fi=>"http://fi.wikipedia.org Finnish",
+ "fiu-vro"=>"http://fiu-vro.wikipedia.org Voro",
+ fiu_vro=>"http://fiu-vro.wikipedia.org Voro",
+ fj=>"http://fj.wikipedia.org Fijian",
+ fo=>"http://fo.wikipedia.org Faroese", # was Faeroese
+ fr=>"http://fr.wikipedia.org French",
+ frp=>"http://frp.wikipedia.org Arpitan",
+ fur=>"http://fur.wikipedia.org Friulian",
+ fy=>"http://fy.wikipedia.org Frisian",
+ ga=>"http://ga.wikipedia.org Irish",
+ gan=>"http://gan.wikipedia.org Gan",
+ gay=>"http://gay.wikipedia.org Gayo",
+ gd=>"http://gd.wikipedia.org Scots Gaelic", # was Scottish Gaelic
+ gl=>"http://gl.wikipedia.org Galician", # was Galego
+ glk=>"http://glk.wikipedia.org Gilaki",
+ gn=>"http://gn.wikipedia.org Guarani",
+ got=>"http://got.wikipedia.org Gothic",
+ gu=>"http://gu.wikipedia.org Gujarati",
+ gv=>"http://gv.wikipedia.org Manx", # was Manx Gaelic
+ ha=>"http://ha.wikipedia.org Hausa",
+ hak=>"http://hak.wikipedia.org Hakka",
+ haw=>"http://haw.wikipedia.org Hawai'ian", # was Hawaiian
+ he=>"http://he.wikipedia.org Hebrew",
+ hi=>"http://hi.wikipedia.org Hindi",
+ hif=>"http://hif.wikipedia.org Fiji Hindi",
+ ho=>"http://ho.wikipedia.org Hiri Motu",
+ hr=>"http://hr.wikipedia.org Croatian",
+ hsb=>"http://hsb.wikipedia.org Upper Sorbian",
+ ht=>"http://ht.wikipedia.org Haitian",
+ hu=>"http://hu.wikipedia.org Hungarian",
+ hy=>"http://hy.wikipedia.org Armenian",
+ hz=>"http://hz.wikipedia.org Herero",
+ ia=>"http://ia.wikipedia.org Interlingua",
+ iba=>"http://iba.wikipedia.org Iban",
+ id=>"http://id.wikipedia.org Indonesian",
+ ie=>"http://ie.wikipedia.org Interlingue",
+ ig=>"http://ig.wikipedia.org Igbo",
+ ii=>"http://ii.wikipedia.org Yi",
+ ik=>"http://ik.wikipedia.org Inupiak",
+ ilo=>"http://ilo.wikipedia.org Ilokano",
+ io=>"http://io.wikipedia.org Ido",
+ is=>"http://is.wikipedia.org Icelandic",
+ it=>"http://it.wikipedia.org Italian",
+ iu=>"http://iu.wikipedia.org Inuktitut",
+ ja=>"http://ja.wikipedia.org Japanese",
+ jbo=>"http://jbo.wikipedia.org Lojban",
+ jv=>"http://jv.wikipedia.org Javanese",
+ ka=>"http://ka.wikipedia.org Georgian",
+ kaa=>"http://kaa.wikipedia.org Karakalpak",
+ kab=>"http://ka.wikipedia.org Kabyle",
+ kaw=>"http://kaw.wikipedia.org Kawi",
+ kg=>"http://kg.wikipedia.org Kongo",
+ ki=>"http://ki.wikipedia.org Kikuyu",
+ kj=>"http://kj.wikipedia.org Kuanyama", # was Otjiwambo
+ kk=>"http://kk.wikipedia.org Kazakh",
+ kl=>"http://kl.wikipedia.org Greenlandic",
+ km=>"http://km.wikipedia.org Khmer", # was Cambodian
+ kn=>"http://kn.wikipedia.org Kannada",
+ ko=>"http://ko.wikipedia.org Korean",
+ kr=>"http://kr.wikipedia.org Kanuri",
+ ks=>"http://ks.wikipedia.org Kashmiri",
+ ksh=>"http://ksh.wikipedia.org Ripuarian",
+ ku=>"http://ku.wikipedia.org Kurdish",
+ kv=>"http://kv.wikipedia.org Komi",
+ kw=>"http://kw.wikipedia.org Cornish", # was Kornish
+ ky=>"http://ky.wikipedia.org Kirghiz",
+ la=>"http://la.wikipedia.org Latin",
+ lad=>"http://lad.wikipedia.org Ladino",
+ lb=>"http://lb.wikipedia.org Luxembourgish", # was Letzeburgesch
+ lbe=>"http://lbe.wikipedia.org Lak",
+ lg=>"http://lg.wikipedia.org Ganda",
+ li=>"http://li.wikipedia.org Limburgish",
+ lij=>"http://lij.wikipedia.org Ligurian",
+ lmo=>"http://lmo.wikipedia.org Lombard",
+ ln=>"http://ln.wikipedia.org Lingala",
+ lo=>"http://lo.wikipedia.org Laotian",
+ ls=>"http://ls.wikipedia.org Latino Sine Flexione",
+ lt=>"http://lt.wikipedia.org Lithuanian",
+ lv=>"http://lv.wikipedia.org Latvian",
+ mad=>"http://mad.wikipedia.org Madurese",
+ mak=>"http://mak.wikipedia.org Makasar",
+ map_bms=>"http://map-bms.wikipedia.org Banyumasan",
+ "map-bms"=>"http://map-bms.wikipedia.org Banyumasan",
+ mdf=>"http://mdf.wikipedia.org Moksha",
+ mg=>"http://mg.wikipedia.org Malagasy",
+ mh=>"http://mh.wikipedia.org Marshallese",
+ mi=>"http://mi.wikipedia.org Maori",
+ min=>"http://min.wikipedia.org Minangkabau",
+ minnan=>"http://minnan.wikipedia.org Minnan",
+ mk=>"http://mk.wikipedia.org Macedonian",
+ ml=>"http://ml.wikipedia.org Malayalam",
+ mn=>"http://mn.wikipedia.org Mongolian",
+ mo=>"http://mo.wikipedia.org Moldavian",
+ mr=>"http://mr.wikipedia.org Marathi",
+ ms=>"http://ms.wikipedia.org Malay",
+ mt=>"http://mt.wikipedia.org Maltese",
+ mus=>"http://mus.wikipedia.org Muskogee",
+ my=>"http://my.wikipedia.org Burmese",
+ myv=>"http://myv.wikipedia.org Erzya",
+ mzn=>"http://mzn.wikipedia.org Mazandarani",
+ na=>"http://na.wikipedia.org Nauruan", # was Nauru
+ nah=>"http://nah.wikipedia.org Nahuatl",
+ nap=>"http://nap.wikipedia.org Neapolitan",
+ nds=>"http://nds.wikipedia.org Low Saxon",
+ nds_nl=>"http://nds-nl.wikipedia.org Dutch Low Saxon",
+ "nds-nl"=>"http://nds-nl.wikipedia.org Dutch Low Saxon",
+ ne=>"http://ne.wikipedia.org Nepali",
+ new=>"http://new.wikipedia.org Nepal Bhasa",
+ ng=>"http://ng.wikipedia.org Ndonga",
+ nl=>"http://nl.wikipedia.org Dutch",
+ nov=>"http://nov.wikipedia.org Novial",
+ nrm=>"http://nrm.wikipedia.org Norman",
+ nn=>"http://nn.wikipedia.org Nynorsk", # was Neo-Norwegian
+ no=>"http://no.wikipedia.org Norwegian",
+ nv=>"http://nv.wikipedia.org Navajo", # was Avayo
+ ny=>"http://ny.wikipedia.org Chichewa",
+ oc=>"http://oc.wikipedia.org Occitan",
+ om=>"http://om.wikipedia.org Oromo",
+ or=>"http://or.wikipedia.org Oriya",
+ os=>"http://os.wikipedia.org Ossetic",
+ pa=>"http://pa.wikipedia.org Punjabi",
+ pag=>"http://pag.wikipedia.org Pangasinan",
+ pam=>"http://pam.wikipedia.org Kapampangan",
+ pap=>"http://pap.wikipedia.org Papiamentu",
+ pdc=>"http://pdc.wikipedia.org Pennsylvania German",
+ pi=>"http://pi.wikipedia.org Pali",
+ pih=>"http://pih.wikipedia.org Norfolk",
+ pl=>"http://pl.wikipedia.org Polish",
+ pms=>"http://pms.wikipedia.org Piedmontese",
+ ps=>"http://ps.wikipedia.org Pashto",
+ pt=>"http://pt.wikipedia.org Portuguese",
+ qu=>"http://qu.wikipedia.org Quechua",
+ rm=>"http://rm.wikipedia.org Romansh", # was Rhaeto-Romance
+ rmy=>"http://rmy.wikipedia.org Romani",
+ rn=>"http://rn.wikipedia.org Kirundi",
+ ro=>"http://ro.wikipedia.org Romanian",
+ roa_rup=>"http://roa-rup.wikipedia.org Aromanian",
+ "roa-rup"=>"http://roa-rup.wikipedia.org Aromanian",
+ roa_tara=>"http://roa-tara.wikipedia.org Tarantino",
+ "roa-tara"=>"http://roa-tara.wikipedia.org Tarantino",
+ ru=>"http://ru.wikipedia.org Russian",
+ ru_sib=>"http://ru-sib.wikipedia.org Siberian",
+ "ru-sib"=>"http://ru-sib.wikipedia.org Siberian",
+ rw=>"http://rw.wikipedia.org Kinyarwanda",
+ sa=>"http://sa.wikipedia.org Sanskrit",
+ sah=>"http://sah.wikipedia.org Sakha",
+ sc=>"http://sc.wikipedia.org Sardinian",
+ scn=>"http://scn.wikipedia.org Sicilian",
+ sco=>"http://sco.wikipedia.org Scots",
+ sd=>"http://sd.wikipedia.org Sindhi",
+ se=>"http://se.wikipedia.org Northern Sami",
+ sg=>"http://sg.wikipedia.org Sangro",
+ sh=>"http://sh.wikipedia.org Serbo-Croatian",
+ si=>"http://si.wikipedia.org Sinhala", # was Singhalese
+ simple=>"http://simple.wikipedia.org Simple English",
+ sk=>"http://sk.wikipedia.org Slovak",
+ sl=>"http://sl.wikipedia.org Slovene",
+ sm=>"http://sm.wikipedia.org Samoan",
+ sn=>"http://sn.wikipedia.org Shona",
+ so=>"http://so.wikipedia.org Somali", # was Somalian
+ sq=>"http://sq.wikipedia.org Albanian",
+ sr=>"http://sr.wikipedia.org Serbian",
+ srn=>"http://srn.wikipedia.org Sranan",
+ ss=>"http://ss.wikipedia.org Siswati",
+ st=>"http://st.wikipedia.org Sesotho",
+ stq=>"http://stq.wikipedia.org Saterland Frisian",
+ su=>"http://su.wikipedia.org Sundanese",
+ sv=>"http://sv.wikipedia.org Swedish",
+ sw=>"http://sw.wikipedia.org Swahili",
+ szl=>"http://szl.wikipedia.org Silesian",
+ ta=>"http://ta.wikipedia.org Tamil",
+ te=>"http://te.wikipedia.org Telugu",
+ test=>"http://test.wikipedia.org Test",
+ tet=>"http://tet.wikipedia.org Tetum",
+ tg=>"http://tg.wikipedia.org Tajik",
+ th=>"http://th.wikipedia.org Thai",
+ ti=>"http://ti.wikipedia.org Tigrinya",
+ tk=>"http://tk.wikipedia.org Turkmen",
+ tl=>"http://tl.wikipedia.org Tagalog",
+ tlh=>"http://tlh.wikipedia.org Klingon", # was Klignon
+ tn=>"http://tn.wikipedia.org Setswana",
+ to=>"http://to.wikipedia.org Tongan",
+ tokipona=>"http://tokipona.wikipedia.org Tokipona",
+ tpi=>"http://tpi.wikipedia.org Tok Pisin",
+ tr=>"http://tr.wikipedia.org Turkish",
+ ts=>"http://ts.wikipedia.org Tsonga",
+ tt=>"http://tt.wikipedia.org Tatar",
+ tum=>"http://tum.wikipedia.org Tumbuka",
+ turn=>"http://turn.wikipedia.org Turnbuka",
+ tw=>"http://tw.wikipedia.org Twi",
+ ty=>"http://ty.wikipedia.org Tahitian",
+ udm=>"http://udm.wikipedia.org Udmurt",
+ ug=>"http://ug.wikipedia.org Uighur",
+ uk=>"http://uk.wikipedia.org Ukrainian",
+ ur=>"http://ur.wikipedia.org Urdu",
+ uz=>"http://uz.wikipedia.org Uzbek",
+ ve=>"http://ve.wikipedia.org Venda", # was Lushaka
+ vec=>"http://vec.wikipedia.org Venetian",
+ vi=>"http://vi.wikipedia.org Vietnamese",
+ vls=>"http://vls.wikipedia.org West Flemish",
+ vo=>"http://vo.wikipedia.org Volap&uuml;k",
+ wa=>"http://wa.wikipedia.org Walloon",
+ war=>"http://war.wikipedia.org Waray-Waray",
+ wo=>"http://wo.wikipedia.org Wolof",
+ wuu=>"http://wuu.wikipedia.org Wu",
+ xal=>"http://xal.wikipedia.org Kalmyk",
+ xh=>"http://xh.wikipedia.org Xhosa",
+ yi=>"http://yi.wikipedia.org Yiddish",
+ yo=>"http://yo.wikipedia.org Yoruba",
+ za=>"http://za.wikipedia.org Zhuang",
+ zea=>"http://zea.wikipedia.org Zealandic",
+ zh=>"http://zh.wikipedia.org Chinese",
+ zh_min_nan=>"http://zh-min-nan.wikipedia.org Min Nan",
+ "zh-min-nan"=>"http://zh-min-nan.wikipedia.org Min Nan",
+ zh_classical=>"http://zh-classical.wikipedia.org Classical Chinese",
+ "zh-classical"=>"http://zh-classical.wikipedia.org Classical Chinese",
+ zh_yue=>"http://zh-yue.wikipedia.org Cantonese",
+ "zh-yue"=>"http://zh-yue.wikipedia.org Cantonese",
+ zu=>"http://zu.wikipedia.org Zulu",
+ zz=>"&nbsp; All&nbsp;languages",
+ zzz=>"&nbsp; All&nbsp;languages except English"
+ );
+
+ foreach $key (keys %wikipedias)
+ {
+ my $wikipedia = $wikipedias {$key} ;
+ $out_urls {$key} = $wikipedia ;
+ $out_languages {$key} = $wikipedia ;
+ $out_urls {$key} =~ s/(^[^\s]+).*$/$1/ ;
+ $out_languages {$key} =~ s/^[^\s]+\s+(.*)$/$1/ ;
+ $out_article {$key} = "http://en.wikipedia.org/wiki/" . $out_languages {$key} . "_language" ;
+ $out_article {$key} =~ s/ /_/g ;
+ $out_urls {$key} =~ s/(^[^\s]+).*$/$1/ ;
+ }
+}
+
+# copied from WikiReports_EN.pl
+sub InitReportNames
+{
+ @out_report_descriptions = (
+ "Contributors",
+ "New editors",
+ "Active editors",
+ "Very active editors",
+ "Article count (official)",
+ "Article count (alternate)",
+ "New articles per day",
+ "Edits per article",
+ "Bytes per article",
+ "Articles over 0.5 Kb",
+ "Articles over 2 Kb",
+ "Edits per month",
+ "Database size",
+ "Words",
+ "Internal links",
+ "Links to other Wikipedias",
+ "Binaries",
+ "External links",
+ "Redirects",
+ "Page requests per day",
+ "Visits per day",
+ "Overview recent months"
+ ) ;
+}
+

Added: trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutputMisc.pl
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutputMisc.pl (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/AnalyticsPrepWikiCountsOutputMisc.pl 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,336 @@
+#!/usr/local/bin/perl
+
+# 2012-06-04 renamed from AnalyticsPrepWikiCountsOutput.pl
+
+# Copyright (C) 2011 Wikimedia Foundation
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 2
+# as published by the Free Software Foundation.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details, at
+# http://www.fsf.org/licenses/gpl.html
+
+# Author:
+# Erik Zachte, email ezachte [at] wikimedia
+# loosely based on predecessor
+# http://svn.wikimedia.org/viewvc/mediawiki/trunk/wikistats/reportcard/ReportCardExtractWikiCountsOutput.pl
+
+# Functionality:
+# tba
+
+# Parameters:
+# tba
+
+# Output:
+# updated csv file for import in MySQL
+
+# http://svn.wikimedia.org/viewvc/mediawiki/trunk/wikistats/analytics/
+
+ use Getopt::Std ;
+
+ $true = 1 ;
+ $false = 0 ;
+
+ @projects = ('wb','wk','wn','wp','wq','ws','wv','wx','commons','*') ;
+
+ $file_csv_monthly_data = "StatisticsMonthly.csv" ;
+ $file_csv_user_activity_spread = "StatisticsUserActivitySpread.csv" ;
+ $file_csv_analytics_in = "analytics_in_wikistats.csv" ;
+
+ &ParseArguments ;
+ &ReadStatisticsMonthly ;
+ &FindLargestWikis ;
+ &WriteMonthlyData ;
+
+ print "\nReady\n\n" ;
+ exit ;
+
+sub ParseArguments
+{
+ print "ParseArguments\n" ;
+ my (@options, $arguments) ;
+
+ getopt ("io", \%options) ;
+
+ foreach $arg (sort keys %options)
+ { $arguments .= " -$arg " . $options {$arg} . "\n" ; }
+ print ("\nArguments\n$arguments\n") ;
+
+ if (! -d '/mnt/') # EZ test
+ {
+ $path_in = "c:/\@ wikimedia/# out bayes" ;
+ $path_out = "c:/MySQL/analytics" ;
+ }
+ else
+ {
+ die ("Specify input folder for projectcounts files as: -i path") if (! defined ($options {"i"})) ;
+ die ("Specify output folder as: -o path'") if (! defined ($options {"o"})) ;
+
+ $path_in = $options {"i"} ;
+ $path_out = $options {"o"} ;
+ }
+
+ die "Input folder '$path_in' does not exist" if (! -d $path_in) ;
+ die "Output folder '$path_out' does not exist" if (! -d $path_out) ;
+
+ print "Input folder: $path_in\n" ;
+ print "Output folder: $path_out\n\n" ;
+
+ $file_csv_out = "$path_out/analytics_in_wikistats.csv" ;
+}
+
+sub ReadStatisticsMonthly
+{
+ print "ReadStatisticsMonthly\n" ;
+ &ReadStatisticsMonthlyForProject ("wb") ;
+ &ReadStatisticsMonthlyForProject ("wk") ;
+ &ReadStatisticsMonthlyForProject ("wn") ;
+ &ReadStatisticsMonthlyForProject ("wp") ;
+ &ReadStatisticsMonthlyForProject ("wq") ;
+ &ReadStatisticsMonthlyForProject ("ws") ;
+ &ReadStatisticsMonthlyForProject ("wv") ;
+ &ReadStatisticsMonthlyForProject ("wx") ;
+
+# &ReadStatisticsPerBinariesExtensionCommons ;
+}
+
+sub ReadStatisticsMonthlyForProject
+{
+ my $project = shift;
+ $all_projects = "*" ;
+
+ my $file_csv_in_1 = "$path_in/csv_$project/$file_csv_monthly_data" ;
+ my $file_csv_in_2 = "$path_in/csv_$project/$file_csv_user_activity_spread" ;
+
+ if (! -e $file_csv_in_1)
+ { &Abort ("Input file '$file_csv_in_1' not found") ; }
+ if (! -e $file_csv_in_2)
+ { &Abort ("Input file '$file_csv_in_2' not found") ; }
+
+ my $yyyymm ;
+
+ print "Read '$file_csv_in_1'\n" ;
+ open CSV_IN, '<', $file_csv_in_1 ;
+ while ($line = <CSV_IN>)
+ {
+ chomp $line ;
+ ($language,$date,$counts) = split (',', $line, 3) ;
+ @fields = split (',', $counts) ;
+
+ next if ! &AcceptWiki ($project,$language) ;
+
+ ($month,$day,$year) = split ('\/', $date) ;
+ $yyyymm = sprintf ("%04d-%02d", $year, $month) ;
+
+ foreach $field (@fields)
+ {
+ if ($field eq '-')
+ { $field = 0 ; }
+ }
+
+ $data = $fields [0] . ',' . # contributors all time
+ $fields [1] . ',' . # new contributors
+ 'data2,' . # place holder for more data, to be inserted later
+ $fields [4] . ',' . # articles
+ $fields [6] . ',' . # articles new per day
+ $fields [9] . ',' . # larger than 0.5 kB
+ $fields [10] . ',' . # larger than 2.0 kB
+ $fields [7] . ',' . # mean edits per article
+ $fields [8] . ',' . # mean bytes per article
+ $fields [11] . ',' . # edits
+ $fields [12] . ',' . # size in bytes
+ $fields [13] . ',' . # size in words
+ $fields [14] . ',' . # links internal
+ $fields [15] . ',' . # links interwiki
+ $fields [16] . ',' . # links images
+ $fields [17] . ',' . # links external
+ $fields [18] ; # redirects
+
+ $data1 {"$project,$language,$yyyymm"} = $data ;
+ }
+ close CSV_IN ;
+
+ # now read (very) active editors from newer more accurate file (split data for reg users and bots, unlike StatisticsMonthly.csv)
+
+ print "Read '$file_csv_in_2'\n" ;
+ open CSV_IN, '<', $file_csv_in_2 ;
+ while ($line = <CSV_IN>)
+ {
+ chomp $line ;
+ ($language,$date,$reguser_bot,$group,@counts) = split (',', $line) ;
+
+ next if ! &AcceptWiki ($project,$language) ;
+
+ if ($reguser_bot ne "R") { next ; } # R: reg user, B: bot
+ if ($group ne "A") { next ; } # A: articles, T: talk pages, O: other namespaces
+
+ ($month,$day,$year) = split ('\/', $date) ;
+ $yyyymm = sprintf ("%04d-%02d", $year, $month) ;
+ $months {$yyyymm} ++ ;
+# print "YYYYMM $yyyymm\n" ;
+
+ # data have been collected in WikiCountsProcess.pm and been written in WikiCountsOutput.pm
+ # count user with over x edits
+ # threshold starting with a 3 are 10xSQRT(10), 100xSQRT(10), 1000xSQRT(10), etc
+ # @thresholds = (1,3,5,10,25,32,50,100,250,316,500,1000,2500,3162,5000,10000,25000,31623,50000,100000,250000,316228,500000,1000000,2500000,3162278,500000,10000000,25000000,31622777,5000000,100000000) ;
+ $edits_ge_5 = @counts [2] > 0 ? @counts [2] : 0 ;
+ $edits_ge_25 = @counts [4] > 0 ? @counts [4] : 0 ;
+ $edits_ge_100 = @counts [7] > 0 ? @counts [7] : 0 ;
+ $data2 {"$project,$language,$yyyymm"} = "$edits_ge_5,$edits_ge_25,$edits_ge_100" ;
+
+ $total_edits_ge_5 {"$project,$language"} += $edits_ge_5 ;
+ $total_edits_ge_25 {"$project,$language"} += $edits_ge_25 ;
+ $total_edits_ge_100 {"$project,$language"} += $edits_ge_100 ;
+
+ # prep string with right amount of comma's
+ if ($data2_default eq '')
+ {
+ $data2_default = $data2 {"$project,$language,$yyyymm"} ;
+ $data2_default =~ s/[^,]+/0/g ;
+ }
+ }
+ close CSV_IN ;
+}
+
+#sub ReadStatisticsPerBinariesExtensionCommons
+#{
+# my $file_csv_in = "$path_in/csv_wx/StatisticsPerBinariesExtension.csv" ;
+# my $mmax = -1 ;
+
+# if (! -e $file_csv_in)
+# { &Abort ("Input file '$file_csv_in' not found") ; }
+
+# print "Read '$file_csv_in'\n" ;
+# open CSV_IN, '<', $file_csv_in ;
+# while ($line = <CSV_IN>)
+# {
+# chomp $line ;
+# ($language,$date,$counts) = split (',', $line, 3) ;
+
+# if ($language ne "commons") { next ; }
+
+# if ($date eq "00/0000")
+# {
+# @fields = split (',', $counts) ;
+# $field_ndx = 0 ;
+# foreach $field (@fields)
+# {
+# $ext_cnt {-1}{$field_ndx} = $field ;
+# # print "EXT_CNT $field_ndx : $field\n" ;
+# $field_ndx ++ ;
+# }
+# next ;
+# }
+
+# ($month,$year) = split ('\/', $date) ;
+# my $m = &months_since_2000_01 ($year,$month) ;
+# next if $m < $m_start ;
+
+# if ($m > $mmax)
+# { $mmax = $m ; }
+
+# @fields = split (',', $counts) ;
+# $field_ndx = 0 ;
+# foreach $field (@fields)
+# {
+# $ext_cnt {$m}{$field_ndx} = $field ;
+# $ext_tot {$m} += $field ;
+# $field_ndx ++ ;
+# }
+# }
+# close CSV_IN ;
+
+# %ext_cnt_mmax = %{$ext_cnt {$mmax}} ;
+# @ext_cnt_mmax = (sort {$ext_cnt_mmax {$b} <=> $ext_cnt_mmax {$a}} keys %ext_cnt_mmax) ;
+
+# $extcnt = 0 ;
+# foreach $extndx (@ext_cnt_mmax)
+# {
+# # print "$extndx < ${ext_cnt {-1}{$extndx}} > : ${ext_cnt_mmax {$extndx}}\n" ;
+# push @extndxs, $extndx ;
+# if ($extcnt++ >= 9) { last ; }
+# }
+#}
+
+sub FindLargestWikis
+{
+ print "FindLargestWikis\n" ;
+ print "Largest projects (most accumulated very active editors):\n";
+ @total_edits_ge_100 = sort {$total_edits_ge_100 {$b} <=> $total_edits_ge_100 {$a}} keys %total_edits_ge_100 ;
+ $rank = 0 ;
+ foreach $project_language (@total_edits_ge_100)
+ {
+ $largest_projects {$project_language} = $rank++ ;
+ print "$project_language," ;
+ last if $rank > 10 ;
+ }
+ print "\n\n" ;
+
+ foreach $yyyymm (sort keys %months)
+ {
+ next if $yyyymm lt '2011' ;
+ foreach $project_language (keys %largest_projects)
+ {
+ ($project,$language) = split (',', $project_language) ;
+ if ($data2 {"$project,$language,$yyyymm"} eq '')
+ {
+ print "No data yet for large wiki $project_language for $yyyymm-> skip month $yyyymm\n" ;
+ $months {$yyyymm} = 0 ;
+ }
+ }
+ }
+}
+
+sub WriteMonthlyData
+{
+ print "WriteMonthlyData\n" ;
+ my $file_csv_out = "$path_out/$file_csv_analytics_in" ;
+ open CSV_OUT, '>', $file_csv_out ;
+ foreach $project_wiki_month (sort keys %data1)
+ {
+ ($project,$wiki,$yyyymm) = split (',', $project_wiki_month) ;
+
+ # recent month misses on eor more large wikis?
+ next if $months {$yyyymm} == 0 ;
+
+ $data1 = $data1 {$project_wiki_month} ;
+ $data2 = $data2 {$project_wiki_month} ;
+ if ($data2 eq '')
+ {
+ print "Editor data missing for $project_wiki_month\n" ;
+ $data2 = $data2_default ;
+ }
+ $data1 =~ s/data2/$data2/ ; # insert rather than append to have all editor fields close together
+ print CSV_OUT "$project_wiki_month,$data1\n" ;
+ }
+ $total_edits_ge_5 {"$project,*,$yyyymm"} += $edits_ge_5 ;
+ $total_edits_ge_25 {"$project,*,$yyyymm"} += $edits_ge_25 ;
+ $total_edits_ge_100 {"$project,*,$yyyymm"} += $edits_ge_100 ;
+ close CSV_OUT ;
+}
+
+sub AcceptWiki
+{
+ my ($project,$language) = @_ ;
+
+ return $false if $language eq 'commons' and $project ne 'wx' ; # commons also in wikipedia csv files (bug, hard to cleanup, just skip)
+ return $false if $language eq 'sr' and $project eq 'wn' ; # ignore insane bot spam on
+ return $false if $language =~ /mania|team|comcom|closed|chair|langcom|office|searchcom|sep11|nostalgia|stats|test/i ;
+
+ return $false if $language =~ /^(?:dk|tlh|ru_sib)$/ ; # dk=dumps exist(ed?) but site not, tlh=Klignon, ru-sib=Siberian
+ return $false if $project eq 'wk' and ($language eq "als" or $language eq "tlh") ;
+
+ return $true ;
+}
+
+sub Abort
+{
+ my $msg = shift ;
+ print "$msg\nExecution aborted." ;
+ # to do: log also to file
+ exit ;
+}
+
+

Added: trunk/wikistats/MetricsReporting/analytics/README
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/README (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/README 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1 @@
+2012-06 files used at stat1:/a/wikstats/analytics
\ No newline at end of file

Added: trunk/wikistats/MetricsReporting/analytics/analytics_create_and_load_from_csv.txt
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/analytics_create_and_load_from_csv.txt (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/analytics_create_and_load_from_csv.txt 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,179 @@
+-- make sure to delete output files *test*.csv first if any exist (MySQL on purpose forbids overwrite)
+
+-- tables implemented:
+-- comscore
+-- comscore_regions
+-- wikistats
+-- page_views
+-- language names
+-- binaries
+
+-- more tables planned (O= optional, not needed for report card stage)
+-- project_names
+-- O edits per project_code, per language, per month, per normalization type (Y/N), editor type (manual, anonymous, bot), namespace group (articles, talk pages, other)
+-- O editors per project_code, per language, per month, per normalization type (Y/N), editor type (manual, anonymous, bot), namespace group (articles, talk pages, other)
+
+
+-- open issues:
+-- only store basic data in database and calculate all aggregates on the fly or do some aggragation before hand ? (e.g. count for mobile / non-mobile / ==> total of both ? <==)
+-- for binaries, store one extension type per row? (future proof, more work to query), or a selected few as columns? (one row only needed per month)
+
+-- Create database and two tables from scratch
+DROP DATABASE IF EXISTS `analytics` ;
+
+CREATE DATABASE `analytics` ;
+
+USE `analytics` ;
+
+CREATE TABLE `comscore` (
+ `date` date NOT NULL,
+ `country_code` varchar (3),
+ `region_code` varchar (3),
+ `web_property` varchar (20),
+ `project_code` varchar (10),
+ `reach` decimal (4,1) DEFAULT NULL,
+ `visitors` decimal (15) DEFAULT NULL,
+ PRIMARY KEY (date,country_code,region_code,project_code,web_property),
+ KEY (`country_code`)
+) ;
+
+CREATE TABLE `comscore_regions` (
+ `region_code` varchar (2),
+ `report_language` varchar (10),
+ `region_name` varchar (18),
+ PRIMARY KEY (report_language,region_code)
+) ;
+
+CREATE TABLE `wikistats` (
+ `date` date NOT NULL,
+ `project_code` varchar (10),
+ `language_code` varchar (15),
+ `editors_all_time` int (10) DEFAULT NULL,
+ `editors_new` int (7) DEFAULT NULL,
+ `editors_ge_5` int (7) DEFAULT NULL,
+ `editors_ge_25` int (7) DEFAULT NULL,
+ `editors_ge_100` int (7) DEFAULT NULL,
+ `articles` int (12) DEFAULT NULL,
+ `articles_new_per_day` int (9) DEFAULT NULL,
+ `articles_over_bytes_500` int (12) DEFAULT NULL,
+ `articles_over_bytes_2000` int (12) DEFAULT NULL,
+ `edits_per_article` decimal (9,1) DEFAULT NULL,
+ `bytes_per_article` decimal (9,1) DEFAULT NULL,
+ `edits` int (12) DEFAULT NULL,
+ `size_in_bytes` int (15) DEFAULT NULL,
+ `size_in_words` int (15) DEFAULT NULL,
+ `links_internal` int (15) DEFAULT NULL,
+ `links_interwiki` int (15) DEFAULT NULL,
+ `links_image` int (15) DEFAULT NULL,
+ `links_external` int (15) DEFAULT NULL,
+ `redirects` int (15) DEFAULT NULL,
+ PRIMARY KEY (date,project_code,language_code)
+) ;
+
+CREATE TABLE `page_views` (
+ `date` date NOT NULL,
+ `project_code` char (2),
+ `language_code` char (15),
+ `views_non_mobile_raw` bigint (15),
+ `views_mobile_raw` bigint (15),
+ `views_non_mobile_normalized` bigint (15),
+ `views_mobile_normalized` bigint (15),
+ `views_raw` bigint (15),
+ `views_normalized` bigint (15),
+ PRIMARY KEY (date,project_code,language_code)
+) ;
+
+CREATE TABLE `language_names` (
+ `report_language` varchar (15),
+ `language_code` varchar (15),
+ `language_name` varchar (50),
+ PRIMARY KEY (report_language,language_code)
+) ;
+
+CREATE TABLE `binaries` (
+ `date` date NOT NULL,
+ `project_code` char (2),
+ `language_code` char (15),
+ `extension` varchar (10),
+ `binaries` bigint (15),
+ PRIMARY KEY (date,project_code,language_code,extension)
+) ;
+
+CREATE TABLE `offline` (
+ `date` date NOT NULL,
+ `readers` bigint (12),
+ PRIMARY KEY (date,readers)
+) ;
+
+-- SHOW TABLES ;
+-- DESCRIBE comscore ;
+-- DESCRIBE comscore_regions ;
+-- DESCRIBE wikistats ;
+-- DESCRIBE page_views ;
+-- DESCRIBE language_names ;
+-- DESCRIBE binaries ;
+-- DESCRIBE offline ;
+
+-- Database Manipulation
+-- Obviously in real world this is a separate script
+
+LOAD DATA LOCAL INFILE 'analytics_in_comscore.csv'
+ INTO TABLE comscore
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (@date,country_code,region_code,web_property,project_code,reach,visitors)
+ SET date = last_day (str_to_date(concat (@date,'-01'),'%Y-%m-%d')) ;
+
+LOAD DATA LOCAL INFILE 'analytics_in_comscore_regions.csv'
+ INTO TABLE comscore_regions
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (report_language,region_code,region_name) ;
+
+LOAD DATA LOCAL INFILE 'analytics_in_wikistats.csv'
+ INTO TABLE wikistats
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (project_code,language_code,@date,editors_all_time,editors_new,editors_ge_5,editors_ge_25,editors_ge_100,articles,articles_new_per_day,articles_over_bytes_500,articles_over_bytes_2000,edits_per_article,bytes_per_article,edits,size_in_bytes,size_in_words,links_internal,links_interwiki,links_image,links_external,redirects)
+ SET date = last_day (str_to_date(concat (@date,'-01'),'%Y-%m-%d')) ;
+
+LOAD DATA LOCAL INFILE 'analytics_in_page_views.csv'
+ INTO TABLE page_views
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (project_code,language_code,@date,views_non_mobile_raw,views_mobile_raw,views_non_mobile_normalized,views_mobile_normalized,views_raw,views_normalized)
+ SET date = last_day (str_to_date(concat (@date,'-01'),'%Y-%m-%d')) ;
+
+
+LOAD DATA LOCAL INFILE 'analytics_in_language_names.csv'
+ INTO TABLE language_names
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (report_language,language_code,language_name) ;
+
+LOAD DATA LOCAL INFILE 'analytics_in_binaries.csv'
+ INTO TABLE binaries
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (project_code,language_code,@date,extension,binaries)
+ SET date = last_day (str_to_date(concat (@date,'-01'),'%Y-%m-%d')) ;
+
+LOAD DATA LOCAL INFILE 'analytics_in_offline.csv'
+ INTO TABLE offline
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (@date,readers)
+ SET date = last_day (str_to_date(concat (@date,'-01'),'%Y-%m-%d')) ;
+
+
+-- show contents (debugging only)
+-- SELECT * FROM comscore ;
+-- SELECT * FROM comscore_regions ;
+-- SELECT * FROM wikistats ;
+-- SELECT * FROM page_views ;
+-- SELECT * FROM language_names ;
+-- SELECT * FROM binaries
+-- WHERE project_code = 'commons' ;
+ SELECT * FROM offline ;
+
+

Added: trunk/wikistats/MetricsReporting/analytics/analytics_generate_csv_files.sh
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/analytics_generate_csv_files.sh (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/analytics_generate_csv_files.sh 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,56 @@
+#!/bin/sh
+
+# Prepare several csv files, ready for importing into analytics database
+# All generated files have _in_ in name signalling these contain data ready for importing into database
+# One input record corresponds to one database record
+
+ulimit -v 8000000
+
+log="analytics_generate_csv_files_log.txt"
+
+# www.walkernews.net/2007/06/03/date-arithmetic-in-linux-shell-scripts
+yyyymm=$(date -d "2 month ago" +"%Y-%m")
+echo process data up to $yyyymm and write to rc-$yyyymm.zip
+
+clear
+cd /a/wikistats/analytics
+
+# AnalyticsPrepBinariesData.pl read counts for binaries which were generated by wikistats
+# and which reside in /a/wikistats/csv_[project code]/StatisticsPerBinariesExtension.csv
+# It filters and reorganizes data and produces analytics_in_binaries.csv
+# Output csv contains: project code, language, month, extension name, count
+
+perl AnalyticsPrepBinariesData.pl -i /a/wikistats/csv/ -o /a/wikistats/analytics/ | tee $log | cat
+
+# AnalyticsPrepComscoreData.pl scans /a/analytics/comscore for newest comScore csv files (with data for last 14 months)
+# parses those csv files, adds/replaces data from these csv files into master files (containing full history)
+# and generates input csv file analytics_in_comscore.csv ready for importing into database
+#
+# note : these csv files were manually downloaded from http://mymetrix.comscore.com/app/report.aspx
+# and given more descriptive names, script finds newest files based on partial name search
+#
+# -r replace (default is add only)
+# -i input folder, contains manually downloaded csv files from comScore (or xls files manually converted to csv)
+# -m master files with full history
+# -o output csv file, with reach per region, UV's per region and UV's per top web property, ready for import into database
+
+perl AnalyticsPrepComscoreData.pl -r -i /a/wikistats/analytics/comscore -m /a/wikistats/analytics -o /a/wikistats/analytics | tee -a $log | cat
+
+# AnalyticsPrepWikiCountsOutput.pl reads a plethora of fields from several csv files from wikistats process
+# It filters and reorganizes data and produces analytics_in_wikistats.csv, ready for import into analytics database
+
+perl AnalyticsPrepWikiCountsOutputMisc.pl -i /a/wikistats/csv/ -o /a/wikistats/analytics | tee -a $log | cat
+perl AnalyticsPrepWikiCountsOutputCore.pl -i /a/wikistats/csv/ -o /a/wikistats/analytics -m $yyyymm | tee -a $log | cat
+
+# analytics_in_page_views.csv is written daily as part of WikiCountsSummarizeProjectCounts.pl
+# part of (/home/ezachte/pageviews_monthly.sh job)
+# which processes hourly projectcounts files (per wiki page view totals for one hour) from http://dammit.lt/wikistats
+# and generates several files on different aggregation levels
+# only action here is to copy data to this folder to have everything in one place
+# note: unlike folder name suggests this file contains stats for all projects
+
+cp /a/wikistats/csv/csv_wp/analytics_in_page_views.csv . | tee -a $log | cat
+cp /a/wikistats/csv/csv_wp/wikilytics_in_pageviews.csv . | tee -a $log | cat
+
+zip rc-$yyyymm.zip wikilytics*.csv
+

Added: trunk/wikistats/MetricsReporting/analytics/analytics_generate_csv_files_log.txt
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/analytics_generate_csv_files_log.txt (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/analytics_generate_csv_files_log.txt 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,1189 @@
+Input folder: /a/wikistats/csv/
+Output folder: /a/wikistats/analytics/
+
+Write file '/a/wikistats/analytics//analytics_in_binaries.csv'
+Read '/a/wikistats/csv//csv_wb/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats/csv//csv_wk/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats/csv//csv_wn/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats/csv//csv_wp/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats/csv//csv_wq/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats/csv//csv_ws/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats/csv//csv_wv/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats/csv//csv_wx/StatisticsPerBinariesExtension.csv'
+
+
+Ready
+
+Mode is replace (specify '-r' for replace)
+
+Directories:
+Input (new comScore data): '/a/wikistats/analytics/comscore'
+Master files (full history): '/a/wikistats/analytics'
+Output (database feed): '/a/wikistats/analytics'
+
+ReadMasterComscoreDataReachPerRegion
+
+Regions found: Europe,North America,Latin America,World-Wide,Middle East - Africa,India,Asia Pacific,China
+UpdateMasterFileFromRecentComscoreData
+
+White list: Europe,North America,Latin America,World-Wide,Middle East - Africa,India,Asia Pacific,China
+
+Latest comscore master file is 0 days old: 'history_comscore_reach_regions.csv'
+Latest comscore update file is 7 days old: 'Multi-Country Media Trend, % reach by region_(Feb 10 - Apr 12).csv'
+Data found: 2011-02,NA = 39.4
+Data found: 2011-03,NA = 40.1
+Data found: 2011-04,NA = 38
+Data found: 2011-05,NA = 39.5
+Data found: 2011-06,NA = 39.1
+Data found: 2011-07,NA = 40.5
+Data found: 2011-08,NA = 42.4
+Data found: 2011-09,NA = 44.4
+Data found: 2011-10,NA = 45.4
+Data found: 2011-11,NA = 44.7
+Data found: 2011-12,NA = 44.8
+Data found: 2012-01,NA = 47.3
+Data found: 2012-02,NA = 46.3
+Data found: 2012-03,NA = 45.2
+Data found: 2012-04,NA = 45.2
+Data found: 2011-02,EU = 41.6
+Data found: 2011-03,EU = 43.1
+Data found: 2011-04,EU = 40.8
+Data found: 2011-05,EU = 44
+Data found: 2011-06,EU = 41.7
+Data found: 2011-07,EU = 39.6
+Data found: 2011-08,EU = 40.9
+Data found: 2011-09,EU = 44.6
+Data found: 2011-10,EU = 47
+Data found: 2011-11,EU = 47.1
+Data found: 2011-12,EU = 45.5
+Data found: 2012-01,EU = 48.1
+Data found: 2012-02,EU = 46.4
+Data found: 2012-03,EU = 46.2
+Data found: 2012-04,EU = 44.8
+Data found: 2011-02,LA = 36.3
+Data found: 2011-03,LA = 41.3
+Data found: 2011-04,LA = 38.7
+Data found: 2011-05,LA = 41.6
+Data found: 2011-06,LA = 39.6
+Data found: 2011-07,LA = 35.4
+Data found: 2011-08,LA = 43
+Data found: 2011-09,LA = 45.5
+Data found: 2011-10,LA = 45.3
+Data found: 2011-11,LA = 44.2
+Data found: 2011-12,LA = 35
+Data found: 2012-01,LA = 36.3
+Data found: 2012-02,LA = 39.6
+Data found: 2012-03,LA = 44.6
+Data found: 2012-04,LA = 41.8
+Data found: 2011-02,I = 30.6
+Data found: 2011-03,I = 30.4
+Data found: 2011-04,I = 29.6
+Data found: 2011-05,I = 31.4
+Data found: 2011-06,I = 32.1
+Data found: 2011-07,I = 33.6
+Data found: 2011-08,I = 35.4
+Data found: 2011-09,I = 35.6
+Data found: 2011-10,I = 34.9
+Data found: 2011-11,I = 35.4
+Data found: 2011-12,I = 35.1
+Data found: 2012-01,I = 36.4
+Data found: 2012-02,I = 34.6
+Data found: 2012-03,I = 34.5
+Data found: 2012-04,I = 32.3
+Data found: 2011-02,W = 28.3
+Data found: 2011-03,W = 29.6
+Data found: 2011-04,W = 27.9
+Data found: 2011-05,W = 29.9
+Data found: 2011-06,W = 28.9
+Data found: 2011-07,W = 28.2
+Data found: 2011-08,W = 30
+Data found: 2011-09,W = 32
+Data found: 2011-10,W = 33.3
+Data found: 2011-11,W = 33
+Data found: 2011-12,W = 31.7
+Data found: 2012-01,W = 33.2
+Data found: 2012-02,W = 32.8
+Data found: 2012-03,W = 33.3
+Data found: 2012-04,W = 32
+Data found: 2011-02,MA = 25.8
+Data found: 2011-03,MA = 28
+Data found: 2011-04,MA = 25.7
+Data found: 2011-05,MA = 28.5
+Data found: 2011-06,MA = 25.8
+Data found: 2011-07,MA = 24.6
+Data found: 2011-08,MA = 25.9
+Data found: 2011-09,MA = 29
+Data found: 2011-10,MA = 32.2
+Data found: 2011-11,MA = 31.5
+Data found: 2011-12,MA = 31.6
+Data found: 2012-01,MA = 32.9
+Data found: 2012-02,MA = 32.2
+Data found: 2012-03,MA = 32.4
+Data found: 2012-04,MA = 30.9
+Data found: 2011-02,AS = 14.2
+Data found: 2011-03,AS = 14.7
+Data found: 2011-04,AS = 14
+Data found: 2011-05,AS = 15.1
+Data found: 2011-06,AS = 15.2
+Data found: 2011-07,AS = 15.7
+Data found: 2011-08,AS = 16.7
+Data found: 2011-09,AS = 17.4
+Data found: 2011-10,AS = 17.9
+Data found: 2011-11,AS = 17.7
+Data found: 2011-12,AS = 17.4
+Data found: 2012-01,AS = 18.1
+Data found: 2012-02,AS = 17.9
+Data found: 2012-03,AS = 18.4
+Data found: 2012-04,AS = 17.2
+Data found: 2011-02,C = 0.9
+Data found: 2011-03,C = 0.9
+Data found: 2011-04,C = 0.9
+Data found: 2011-05,C = 1.1
+Data found: 2011-06,C = 1.2
+Data found: 2011-07,C = 1.5
+Data found: 2011-08,C = 1.6
+Data found: 2011-09,C = 1.7
+Data found: 2011-10,C = 2.2
+Data found: 2011-11,C = 2.2
+Data found: 2011-12,C = 2.7
+Data found: 2012-01,C = 2.6
+Data found: 2012-02,C = 2.4
+Data found: 2012-03,C = 2.8
+Data found: 2012-04,C = 2
+
+Updates found, rewrite master file 'history_comscore_reach_regions.csv'
+
+ReadMasterComscoreDataVisitorsPerRegion
+
+Regions found: World-Wide,Europe,Asia Pacific,North America,Latin America,Middle East - Africa,India,China
+UpdateMasterFileFromRecentComscoreData
+
+White list: World-Wide,Europe,Asia Pacific,North America,Latin America,Middle East - Africa,India,China
+
+Latest comscore master file is 0 days old: 'history_comscore_UV_regions.csv'
+Latest comscore update file is 7 days old: 'Multi-Country Media Trend, UVs by region_(Feb 10 - Apr 12).csv'
+Data found: 2011-02,W = 379415000
+Data found: 2011-03,W = 400011000
+Data found: 2011-04,W = 380716000
+Data found: 2011-05,W = 411061000
+Data found: 2011-06,W = 399362000
+Data found: 2011-07,W = 393543000
+Data found: 2011-08,W = 422779000
+Data found: 2011-09,W = 454529000
+Data found: 2011-10,W = 476627000
+Data found: 2011-11,W = 474723000
+Data found: 2011-12,W = 457063000
+Data found: 2012-01,W = 482157000
+Data found: 2012-02,W = 475699000
+Data found: 2012-03,W = 489402000
+Data found: 2012-04,W = 473380000
+Data found: 2011-02,EU = 150775000
+Data found: 2011-03,EU = 156913000
+Data found: 2011-04,EU = 149211000
+Data found: 2011-05,EU = 161311000
+Data found: 2011-06,EU = 153883000
+Data found: 2011-07,EU = 146724000
+Data found: 2011-08,EU = 152109000
+Data found: 2011-09,EU = 166505000
+Data found: 2011-10,EU = 176918000
+Data found: 2011-11,EU = 178580000
+Data found: 2011-12,EU = 173496000
+Data found: 2012-01,EU = 184668000
+Data found: 2012-02,EU = 179385000
+Data found: 2012-03,EU = 180827000
+Data found: 2012-04,EU = 176376000
+Data found: 2011-02,AS = 77468000
+Data found: 2011-03,AS = 80997000
+Data found: 2011-04,AS = 77891000
+Data found: 2011-05,AS = 84666000
+Data found: 2011-06,AS = 86539000
+Data found: 2011-07,AS = 90378000
+Data found: 2011-08,AS = 97661000
+Data found: 2011-09,AS = 102341000
+Data found: 2011-10,AS = 106128000
+Data found: 2011-11,AS = 105296000
+Data found: 2011-12,AS = 103851000
+Data found: 2012-01,AS = 108186000
+Data found: 2012-02,AS = 105783000
+Data found: 2012-03,AS = 110909000
+Data found: 2012-04,AS = 104524000
+Data found: 2011-02,NA = 80038000
+Data found: 2011-03,NA = 81918000
+Data found: 2011-04,NA = 78234000
+Data found: 2011-05,NA = 82081000
+Data found: 2011-06,NA = 80753000
+Data found: 2011-07,NA = 83859000
+Data found: 2011-08,NA = 88277000
+Data found: 2011-09,NA = 93193000
+Data found: 2011-10,NA = 96252000
+Data found: 2011-11,NA = 94689000
+Data found: 2011-12,NA = 94413000
+Data found: 2012-01,NA = 99821000
+Data found: 2012-02,NA = 98049000
+Data found: 2012-03,NA = 97001000
+Data found: 2012-04,NA = 96664000
+Data found: 2011-02,LA = 41411000
+Data found: 2011-03,LA = 47583000
+Data found: 2011-04,LA = 45108000
+Data found: 2011-05,LA = 49095000
+Data found: 2011-06,LA = 47203000
+Data found: 2011-07,LA = 42700000
+Data found: 2011-08,LA = 52983000
+Data found: 2011-09,LA = 56613000
+Data found: 2011-10,LA = 57210000
+Data found: 2011-11,LA = 56544000
+Data found: 2011-12,LA = 45235000
+Data found: 2012-01,LA = 47386000
+Data found: 2012-02,LA = 51425000
+Data found: 2012-03,LA = 58584000
+Data found: 2012-04,LA = 55227000
+Data found: 2011-02,MA = 29722000
+Data found: 2011-03,MA = 32599000
+Data found: 2011-04,MA = 30272000
+Data found: 2011-05,MA = 33908000
+Data found: 2011-06,MA = 30985000
+Data found: 2011-07,MA = 29882000
+Data found: 2011-08,MA = 31749000
+Data found: 2011-09,MA = 35877000
+Data found: 2011-10,MA = 40119000
+Data found: 2011-11,MA = 39614000
+Data found: 2011-12,MA = 40068000
+Data found: 2012-01,MA = 42097000
+Data found: 2012-02,MA = 41056000
+Data found: 2012-03,MA = 42081000
+Data found: 2012-04,MA = 40589000
+Data found: 2011-02,I = 12960000
+Data found: 2011-03,I = 13011000
+Data found: 2011-04,I = 12801000
+Data found: 2011-05,I = 13701000
+Data found: 2011-06,I = 14163000
+Data found: 2011-07,I = 14979000
+Data found: 2011-08,I = 15944000
+Data found: 2011-09,I = 16180000
+Data found: 2011-10,I = 16019000
+Data found: 2011-11,I = 16425000
+Data found: 2011-12,I = 16425000
+Data found: 2012-01,I = 17205000
+Data found: 2012-02,I = 18771000
+Data found: 2012-03,I = 19402000
+Data found: 2012-04,I = 18688000
+Data found: 2011-02,C = 2664000
+Data found: 2011-03,C = 2854000
+Data found: 2011-04,C = 2866000
+Data found: 2011-05,C = 3492000
+Data found: 2011-06,C = 3960000
+Data found: 2011-07,C = 4814000
+Data found: 2011-08,C = 5163000
+Data found: 2011-09,C = 5709000
+Data found: 2011-10,C = 7386000
+Data found: 2011-11,C = 7355000
+Data found: 2011-12,C = 8803000
+Data found: 2012-01,C = 8766000
+Data found: 2012-02,C = 7886000
+Data found: 2012-03,C = 9249000
+Data found: 2012-04,C = 6666000
+
+Entities ignored:
+United States
+
+
+Updates found, rewrite master file 'history_comscore_UV_regions.csv'
+
+ReadMasterComscoreDataVisitorsPerProperty
+
+UpdateMasterFileFromRecentComscoreData
+
+White list: Total Internet,Google,Microsoft,Facebook,Wikimedia,Yahoo!,Amazon,Apple,AOL,Ask Network,eBay,CBS,Tencent,Glam Media,Baidu
+
+Latest comscore master file is 0 days old: 'history_comscore_UV_properties.csv'
+Latest comscore update file is 35 days old: 'Top 1000 poperties, UV trend_(Jan 10 - Mar 12).csv'
+Data found: 2011-01,Total Internet = 1333403000
+Data found: 2011-02,Total Internet = 1339496000
+Data found: 2011-03,Total Internet = 1350539000
+Data found: 2011-04,Total Internet = 1362369000
+Data found: 2011-05,Total Internet = 1373976000
+Data found: 2011-06,Total Internet = 1383098000
+Data found: 2011-07,Total Internet = 1394031000
+Data found: 2011-08,Total Internet = 1411178000
+Data found: 2011-09,Total Internet = 1420316000
+Data found: 2011-10,Total Internet = 1431939000
+Data found: 2011-11,Total Internet = 1438877000
+Data found: 2011-12,Total Internet = 1444092000
+Data found: 2012-01,Total Internet = 1450588000
+Data found: 2012-02,Total Internet = 1448160000
+Data found: 2012-03,Total Internet = 1471414000
+Data found: 2011-01,Google = 985289000
+Data found: 2011-02,Google = 976374000
+Data found: 2011-03,Google = 996706000
+Data found: 2011-04,Google = 990970000
+Data found: 2011-05,Google = 1009699000
+Data found: 2011-06,Google = 1023643000
+Data found: 2011-07,Google = 1031584000
+Data found: 2011-08,Google = 1048918000
+Data found: 2011-09,Google = 1066695000
+Data found: 2011-10,Google = 1083316000
+Data found: 2011-11,Google = 1081821000
+Data found: 2011-12,Google = 1092912000
+Data found: 2012-01,Google = 1107467000
+Data found: 2012-02,Google = 1107171000
+Data found: 2012-03,Google = 1139459000
+Data found: 2011-01,Microsoft = 889163000
+Data found: 2011-02,Microsoft = 837039000
+Data found: 2011-03,Microsoft = 895813000
+Data found: 2011-04,Microsoft = 892893000
+Data found: 2011-05,Microsoft = 905454000
+Data found: 2011-06,Microsoft = 882818000
+Data found: 2011-07,Microsoft = 898728000
+Data found: 2011-08,Microsoft = 887917000
+Data found: 2011-09,Microsoft = 914237000
+Data found: 2011-10,Microsoft = 878510000
+Data found: 2011-11,Microsoft = 886560000
+Data found: 2011-12,Microsoft = 897860000
+Data found: 2012-01,Microsoft = 889202000
+Data found: 2012-02,Microsoft = 855114000
+Data found: 2012-03,Microsoft = 874946000
+Data found: 2011-01,Facebook = 675361000
+Data found: 2011-02,Facebook = 676733000
+Data found: 2011-03,Facebook = 692998000
+Data found: 2011-04,Facebook = 697970000
+Data found: 2011-05,Facebook = 713647000
+Data found: 2011-06,Facebook = 734240000
+Data found: 2011-07,Facebook = 736465000
+Data found: 2011-08,Facebook = 752382000
+Data found: 2011-09,Facebook = 769655000
+Data found: 2011-10,Facebook = 788181000
+Data found: 2011-11,Facebook = 792999000
+Data found: 2011-12,Facebook = 794349000
+Data found: 2012-01,Facebook = 805604000
+Data found: 2012-02,Facebook = 790100000
+Data found: 2012-03,Facebook = 805722000
+Data found: 2011-01,Wikimedia = 413957000
+Data found: 2011-02,Wikimedia = 379415000
+Data found: 2011-03,Wikimedia = 400011000
+Data found: 2011-04,Wikimedia = 380716000
+Data found: 2011-05,Wikimedia = 411061000
+Data found: 2011-06,Wikimedia = 399362000
+Data found: 2011-07,Wikimedia = 393543000
+Data found: 2011-08,Wikimedia = 422779000
+Data found: 2011-09,Wikimedia = 454529000
+Data found: 2011-10,Wikimedia = 476627000
+Data found: 2011-11,Wikimedia = 474723000
+Data found: 2011-12,Wikimedia = 457063000
+Data found: 2012-01,Wikimedia = 482157000
+Data found: 2012-02,Wikimedia = 475699000
+Data found: 2012-03,Wikimedia = 489402000
+Data found: 2011-01,Amazon = 258169000
+Data found: 2011-02,Amazon = 233765000
+Data found: 2011-03,Amazon = 271264000
+Data found: 2011-04,Amazon = 268611000
+Data found: 2011-05,Amazon = 284321000
+Data found: 2011-06,Amazon = 282233000
+Data found: 2011-07,Amazon = 288766000
+Data found: 2011-08,Amazon = 302455000
+Data found: 2011-09,Amazon = 319548000
+Data found: 2011-10,Amazon = 338444000
+Data found: 2011-11,Amazon = 350769000
+Data found: 2011-12,Amazon = 366199000
+Data found: 2012-01,Amazon = 368190000
+Data found: 2012-02,Amazon = 353548000
+Data found: 2012-03,Amazon = 361472000
+Data found: 2011-01,Apple = 236347000
+Data found: 2011-02,Apple = 221271000
+Data found: 2011-03,Apple = 234944000
+Data found: 2011-04,Apple = 228366000
+Data found: 2011-05,Apple = 244182000
+Data found: 2011-06,Apple = 246889000
+Data found: 2011-07,Apple = 256743000
+Data found: 2011-08,Apple = 260749000
+Data found: 2011-09,Apple = 264537000
+Data found: 2011-10,Apple = 299360000
+Data found: 2011-11,Apple = 285234000
+Data found: 2011-12,Apple = 295771000
+Data found: 2012-01,Apple = 302250000
+Data found: 2012-02,Apple = 290757000
+Data found: 2012-03,Apple = 306808000
+Data found: 2011-01,CBS = 226135000
+Data found: 2011-02,CBS = 210949000
+Data found: 2011-03,CBS = 234656000
+Data found: 2011-04,CBS = 217686000
+Data found: 2011-05,CBS = 220230000
+Data found: 2011-06,CBS = 214019000
+Data found: 2011-07,CBS = 217962000
+Data found: 2011-08,CBS = 226901000
+Data found: 2011-09,CBS = 242571000
+Data found: 2011-10,CBS = 244382000
+Data found: 2011-11,CBS = 238148000
+Data found: 2011-12,CBS = 245870000
+Data found: 2012-01,CBS = 259109000
+Data found: 2012-02,CBS = 253798000
+Data found: 2012-03,CBS = 270398000
+Data found: 2011-01,Tencent = 208512000
+Data found: 2011-02,Tencent = 207793000
+Data found: 2011-03,Tencent = 214254000
+Data found: 2011-04,Tencent = 216190000
+Data found: 2011-05,Tencent = 220707000
+Data found: 2011-06,Tencent = 229175000
+Data found: 2011-07,Tencent = 237849000
+Data found: 2011-08,Tencent = 243094000
+Data found: 2011-09,Tencent = 245220000
+Data found: 2011-10,Tencent = 249286000
+Data found: 2011-11,Tencent = 248554000
+Data found: 2011-12,Tencent = 248644000
+Data found: 2012-01,Tencent = 252822000
+Data found: 2012-02,Tencent = 249661000
+Data found: 2012-03,Tencent = 264118000
+Data found: 2011-01,Ask Network = 234884000
+Data found: 2011-02,Ask Network = 220019000
+Data found: 2011-03,Ask Network = 236956000
+Data found: 2011-04,Ask Network = 220341000
+Data found: 2011-05,Ask Network = 224060000
+Data found: 2011-06,Ask Network = 212548000
+Data found: 2011-07,Ask Network = 217771000
+Data found: 2011-08,Ask Network = 227266000
+Data found: 2011-09,Ask Network = 240805000
+Data found: 2011-10,Ask Network = 255792000
+Data found: 2011-11,Ask Network = 245416000
+Data found: 2011-12,Ask Network = 239319000
+Data found: 2012-01,Ask Network = 266079000
+Data found: 2012-02,Ask Network = 243983000
+Data found: 2012-03,Ask Network = 263148000
+Data found: 2011-01,Baidu = 190956000
+Data found: 2011-02,Baidu = 192845000
+Data found: 2011-03,Baidu = 202995000
+Data found: 2011-04,Baidu = 201227000
+Data found: 2011-05,Baidu = 211060000
+Data found: 2011-06,Baidu = 215086000
+Data found: 2011-07,Baidu = 225898000
+Data found: 2011-08,Baidu = 231609000
+Data found: 2011-09,Baidu = 237997000
+Data found: 2011-10,Baidu = 243226000
+Data found: 2011-11,Baidu = 241368000
+Data found: 2011-12,Baidu = 246108000
+Data found: 2012-01,Baidu = 250655000
+Data found: 2012-02,Baidu = 252933000
+Data found: 2012-03,Baidu = 262638000
+Data found: 2011-01,eBay = 226079000
+Data found: 2011-02,eBay = 211945000
+Data found: 2011-03,eBay = 218809000
+Data found: 2011-04,eBay = 215085000
+Data found: 2011-05,eBay = 225893000
+Data found: 2011-06,eBay = 223520000
+Data found: 2011-07,eBay = 224511000
+Data found: 2011-08,eBay = 230320000
+Data found: 2011-09,eBay = 232945000
+Data found: 2011-10,eBay = 244368000
+Data found: 2011-11,eBay = 243222000
+Data found: 2011-12,eBay = 241919000
+Data found: 2012-01,eBay = 254801000
+Data found: 2012-02,eBay = 237301000
+Data found: 2012-03,eBay = 241017000
+Data found: 2011-01,AOL = 243275000
+Data found: 2011-02,AOL = 229394000
+Data found: 2011-03,AOL = 247697000
+Data found: 2011-04,AOL = 237258000
+Data found: 2011-05,AOL = 238860000
+Data found: 2011-06,AOL = 231564000
+Data found: 2011-07,AOL = 227626000
+Data found: 2011-08,AOL = 227538000
+Data found: 2011-09,AOL = 220379000
+Data found: 2011-10,AOL = 224822000
+Data found: 2011-11,AOL = 219953000
+Data found: 2011-12,AOL = 221648000
+Data found: 2012-01,AOL = 218280000
+Data found: 2012-02,AOL = 218284000
+Data found: 2012-03,AOL = 221302000
+Data found: 2011-01,Glam Media = 202299000
+Data found: 2011-02,Glam Media = 194173000
+Data found: 2011-03,Glam Media = 204134000
+Data found: 2011-04,Glam Media = 200186000
+Data found: 2011-05,Glam Media = 213528000
+Data found: 2011-06,Glam Media = 203630000
+Data found: 2011-07,Glam Media = 200757000
+Data found: 2011-08,Glam Media = 202836000
+Data found: 2011-09,Glam Media = 206764000
+Data found: 2011-10,Glam Media = 210711000
+Data found: 2011-11,Glam Media = 209031000
+Data found: 2011-12,Glam Media = 216334000
+Data found: 2012-01,Glam Media = 232000000
+Data found: 2012-02,Glam Media = 220460000
+Data found: 2012-03,Glam Media = 212773000
+
+Entities ignored:
+114LA.COM, 120ASK.COM, 123GREETINGS.COM, 1616.NET, 21CN Corporation Limited, 2345.COM, 24H.COM.VN, 265.COM, 2CH.NET, 2DPLAY.COM, 2SHARED.COM, 39.NET, 4SHARED.COM*, 51.COM, 51JOB.COM, 51MOLE.COM, 52PK.COM, 56.COM, 58.COM, 6.CN, 6ARAB.COM, 6RB.COM, 6RBTOP.COM, 7K7K.COM, 8684.CN, 8D.CN, ABC OTV, ABOUTUS.ORG, AL-WLID.COM, ALTOOLS.COM, ARABSEYES.COM, ARCHIVE.ORG, ARD Sites, ARTISTdirect Network, AT&T Inc., AT&T Interactive Network, AVG Technologies, AVS4YOU.COM, Accor, AccuWeather Sites, Acer Inc., Activision Blizzard, Adobe Sites, Adoos Classifieds S.L., Advance Digital, Agricultural Bank of China, Air France-KLM Group, Alibaba.com Corporation, Aljazeera, Allesklar.com AG Sites, AlloCine Sites, Alloy Digital Network, Altice, Amadeus Global Travel Distribution, American Airlines, American Express, AmericanGreetings Property, Ancestry.com Inc., Answers.com Sites, Arcandor AG, Art.com Sites, ArticlesBase.com Sites, Asahi Shimbun, Atrinsic Inc., Avon Products Inc., Axel Springer AG, B5Media, BABYLON.COM, BADONGO.COM, BADOO.COM, BAIXING.COM, BBC Sites, BBVOD.NET, BEEMP3.COM, BIGMIR.NET, BITCOMET.COM, BLOGBUS.COM*, BLOGGUM.COM, BMW AG, BNP Paribas, BOOSJ.COM, BRG8.COM, BTJUNKIE.ORG, BUECHER.DE, BUZZMEDIA, BabyCenter, Bank of America, Barclays Bank, Barnes & Noble, Batanga, Bauer Consumer Media, BearShare (App), Belgacom Group, Belo, Benesse Corporation, Berkshire Hathaway Inc., Best Buy Sites, Bestofmedia Group, Big Fish Games Sites, Big Lead Sports By FSV, Bigpoint, BitComet (App), BitTorrent Network, Blockbuster Inc., Bokee Sites, Bravenet Web Services, Break Media, British Telecommunications, BuscaPe.com Inc., CA.GOV, CANALBLOG.COM*, CBC-Radio Canada Sites, CCLEANER.COM, CCM-Benchmark, CDC Corporation Sites, CDUNIVERSE.COM, CENTERBLOG.NET, CHINA.COM.CN, CITY-DATA.COM, CJ Group, CONDUIT.COM, COOKPAD.COM, COOLCHASER.COM, COOLS4U.COM, COOLSAVINGS.COM, CQVIP.COM, CRI Online Sites, CRSKY.COM, CSDN.NET Sites, CT10000.COM, CYBERLINK.COM, Cablevision, CafeMom Network, CafePress, Camfrog (App), Canoe Network, Canon Inc., Capital One, CareerBuilder LLC, Carrefour Group, Channel4, China Central Television Sites, China Construction Bank, China Economic Net, China Mobile Sites, China Telecom Sites, Chinabyte Technology Co., Chosun Ilbo, Cisco Systems Inc., Citigroup, Citrix Systems Inc., CityGrid Media, Classified Ventures, Clear Channel Online, Clearspring Technologies, Coca-Cola Company, CollegeHumor Media, Compare Infobase Limited, Conde Nast Digital, Connexus, Corel Corporation, Costco Wholesale Corporation, Coupons Inc., Cox Enterprises Inc., CraveOnline (Evolve Media Corp.), Ctrip.com International, CyberAgent, Cyworld, DAILYMOTION.COM, DANGDANG.COM, DEEZER.COM, DEPOSITFILES.COM, DEVIANTART.COM, DIPAN.COM, DIRECT.GOV.UK, DIVSHARE.COM, DLTK Network Sites, DMGT (exc Mail Online), DOMAINTOOLS.COM, DOUBAN.COM, DOWNLOAD3000.COM, DUOTE.COM, DUOTEGAME.COM, DUOWAN.COM, DeNA Co. Ltd., Dell, Delta Airlines, Demand Media, Department of Commerce, Deutsche Bahn, Deutsche Post AG Sites, Deutsche Telekom, DexOne, Directv Sites, Discover Financial Services, Discovery Digital Media Sites, Disney Online, Dogan Online, Dominion Enterprises, Domino's Pizza Inc., Dong-A Ilbo, Dow Jones & Company, Dwango Co. Ltd., E.W. Scripps, EA Online, EASY-SHARE.COM, ED.GOV, ELAANA.COM, EMP3WORLD.COM, ESNIPS.COM, ESPN, EUROPA.EU, EVA.VN, EVERSAVE.COM, EZINEARTICLES.COM, Earthlink, Eastman Kodak, EasyGroup, Ebuzzing, Ekspress Grupp, Encyclopaedia Britannica, Eniro Sites, Estee Lauder Sites, Eutelia, Everyday Health, Excite Japan Sites, Expedia Inc, Experian Interactive, FANBOX.COM, FC2 inc., FILEHIPPO.COM, FILESTUBE.COM, FLASHGAMES247.COM, FRIENDSTER.COM, FUNSHION.COM, Fairfax Media, FedEx, Federated Media Publishing, First Choice Holidays PLC, Forbes Digital, Ford Motor Company, FortuneCity Global Community, ForumCommunity, Fotosearch LLC, Fotovista Group, Fox Interactive Media, France Televisions Interactive, Freechal INC., FrostWire, Fuji Television Network, GAME2.CN, GAMEBTO.COM, GAMES2GIRLS.COM, GAMETOP.COM, GAMEZER.COM, GANJI.COM, GARENA.COM, GGL Media, GMO Internet Group, GN Multicultural (Evolve Media Corp.), GRAAAM.COM, GSN Games, GYYX.CN, GameDuell, GameStop, Gameforge Sites, Gannett Sites, Ganz, Gap Inc., Gawker Media, Geeknet, General Mills, General Motors, Getty Images Inc, GiantRealm, Globe7, Globo, GoNamic GmbH, Gorilla Nation (Evolve Media Corp), Gretech Corp., Groupe Amaury, Groupe Auchan, Groupe Casino, Groupe Credit Agricole, Groupe Figaro, Groupe LDLC.com, Groupe Lagardere, Groupe M6, Groupe PPR, Groupe Pages Jaunes, Groupe SNCF, Groupe Spir Communication, Groupe TF1, Gruner+Jahr Sites, Grupo Abril, Grupo Carso, Grupo Clarin, Grupo Intercom, Grupo Planeta Sites, Grupo Santander, Grupo Televisa, Gruppo Editoriale Espresso, Gruppo Fiat, Gruppo Mediaset, Guardian Media Group (exc The Guardian), Gurunavi, H&R Block, HAWAAWORLD.COM, HC International Inc., HENHAOJI.COM, HEXUN.COM, HI5.COM, HMV, HONGXIU.COM, HSBC, Habbo Network, Hallmark, Hankook Ilbo, Hankyoreh Newspaper, Hatena, HealthCentral, HealthGrades, Healthline, Hearst Corporation, Hearst Digital Media, Hewlett Packard, Hi-Media Group, Hilton Hotels, Hispavista Sites, Hollywood.com Sites, Home Retail Group, HomeAway, Honda, HotChalk, Hubert Burda Media, Hulu, Hutchison Whampoa, Hyundai Kia Automotive Group, IBM, IDG Network, IDT Corporation, IKEA, IKM Internet Kaufmarkt, IMAGEVENUE.COM, INFO.COM*, ING Group, IP138.COM, IRS.GOV, IT&IS Siglo XXI, ITV Sites, Iliad - Free.fr Sites, Imaginova Network, Impress Group, IncrediMail Ltd, Indeed, Independent News & Media, Indian Railways, Infospace Network, InnoGames GmbH, Intel, Intelius, InterContinental Hotels Group, Interactive One, Interdots International, Internet Brands Inc., Internet Initiative Japan, Interpark, Intuit, JCPenney Sites, JEDDAHBIKERS.COM, JEERAN.COM, JMnet (JoongAng Media Network), JPMorgan Chase Property, JSOFTJ.COM, Jabbar Internet Group, Japan Post Group, Job.com Sites, KAIXIN001.COM, KBB.COM, KBS (Korean Broadcasting System), KDDI Corporation, KEENFINDER.COM, KIEV.UA, KINO.TO, KONGREGATE.COM, KOOORA.COM, KOOWO.COM, KPN, KT Group, KUWAITUP.COM, KaiserGames GmbH, Kakaku.com Inc., Kayak.com Network, Kiabi, Kingfisher, Kingsoft Corp, Kipubli, Kohls Corporation, Korea Economic Daily, Kraft Foods, KuGou Inc., Kuaiche.com Sites, L'Oreal Sites, LEGACY.COM*, LEO.ORG, LETITBIT.NET, LG Group, LIMEWIRE.COM, LINKEDIN.COM, LIVESCORE.COM, LOCALBILLING.COM, LOWES.COM, LVMH Sites, La Poste, Le Monde Sites, LeGuide.com Sites, Lee Enterprises Incorporated, Liberty Media Holding Corporation, Limitedbrands, Live Nation Network, Lloyds Banking Group plc, Lojas Americanas, Lotte Group, Lycos Global Search And Media, M5ZN.COM, MADE-IN-CHINA.COM, MANTA.COM, MAPBAR.COM, MBC (Munhwa Broadcasting Corp.), MEDIAFIRE.COM, MEGAUPLOAD.COM, MEGAVIDEO.COM, MERCHANTCIRCLE.COM, METEOFRANCE.COM, METRO Group, MEXAT.COM, MIH Limited, MINICLIP.COM, MININOVA.ORG, MIXI Inc., MONOGRAFIAS.COM, MOZOOK.COM, MSGPLUSLIVE.NET, MULTIPLY.COM, MUSICA.COM, MYHOTCOMMENTS.COM, Macy's Inc., Maeil Business Newspaper, Mainichi Shimbun, Mapsolute, Marriott, Match.com Sites, Mattel Sites, McAfee.com Sites, McClatchy Corporation, McDonald's, Media General Sites, MediaNews Group, Meebo Messenger IM, MeetIC - Match, MercadoLibre, Meredith Women's Network, Metacafe, Miarroba Networks, Michelin Group, Midasplayer.com Ltd., Mio.tv Network, Moneysupermarket.com Financial Group, Monster Inc., Move Inc. Real Estate Sites, MyLife.com Sites, NAMIPAN.COM, NASA.GOV, NEC Corporation, NERO.COM, NETFLIX.COM, NFL Internet Group, NHACCUATUI.COM, NHK Group, NHL Network, NHN Corporation, NIC.IN, NIH, NIKKEI, NK.PL, NOWCOM, NTT Group, National Geographic Sites, National Lampoon Humor Network, Naukri, Nestle, NetEase.com Inc., NetShelter Technology Media, Network 18, New York Times Digital, NewIPs, News Interactive Pty Limited, News International, News World Communications Inc., Nexon Corporation, NextRadio TV, Nextag.com Sites, Nifty Corporation, Nike, Nintendo Co., Nippon Television Network, Nissan, Nokia, Notrefamille.com Sites, OHMYNEWS.COM, OLX Inc., OPENDNS.COM, OPENV.COM*, ORBITDOWNLOADER.COM, OVH Sites, OWNSKIN.COM, OYUNLAR1.COM, OZQ8.COM, Oak Pacific Interactive Sites, Oberon Media Inc, Office Depot, Onlinedown Sites, Opera Software, Oracle Corporation, Orange Sites, Orbitz Worldwide, Oricon, Otto Gruppe, Oversee.net Network, PANDORA.COM, PANDORA.TV, PANET.CO.IL, PBASE.COM, PBS, PCPOP Sites, PChome Online, PConline.cn Sites, PHOTOFUNIA.COM, PHPNUKE.ORG, PICNIK.COM, PIPI.CN, PLAYLIST.COM, POMOHO.COM, PPLive Inc., PPStream Inc., PSA Peugeot Citro├źn, PSICOFXP.COM, PYZAM.COM, PalTalk, Panasonic Group, Pangea Media, PartyGaming, Pearson Education, People's Daily online, Pepsi Co., Photobucket.com LLC, PlasmaNet Sites, PokerStars, Populis, Portugal Telecom, Postmedia Network, PriceGrabber, Priceline.com Incorporated, Prisa, ProSiebenSat1 Sites, Procter & Gamble, Proto Corporation, Publieke Omroep, QIANQIAN.COM, QIDIAN.COM, QQ163.COM, QVOD.COM, Qihoo.com Sites, RAPIDLIBRARY.COM, RARLAB.COM, RAYFILE.COM, RAYLI.COM.CN, RBC Financial Group, RCS Media Group, READNOVEL.COM, REGNOW.COM, ROGERS, RORO44.COM, RTL Group Sites, Rakuten Inc, Rambler Media, RapidShare AG, Reader's Digest Digital Network, Real.com Network, Recruit Co. Ltd., Rediff.com India Ltd, Reed Business Information, Reed Elsevier Group, Research In Motion Limited, Reuters Sites, Rising Corp. Limited, RosBusinessConsulting, Roularta Media Group, Rue du Commerce, Ryanair, SBS (Seoul Broadcasting System), SCRIBD.COM, SEDO Sites, SENDSPACE.COM, SH.CN, SHERV.NET, SHOPLOCAL.COM, SHOUTMIX.COM, SHVOONG.COM, SINA Corporation, SK Group, SLIDESHARE.NET, SNDA Sites, SOFTBANK Group, SOFTPEDIA.COM, SONGS.PK, SPB.RU, SPLINDER.COM, SPRINGERLINK.COM, STARTIMES2.COM, STARTVG.COM, SUMOTORRENT.COM, SWEETIM.COM, Sakura Internet, Samsung Group, Samurai Factory Inc., Sankei Shimbun, Sanoma Group, Schibsted (Anuntis-Infojobs-20minutos), Scripps Networks Interactive Inc., SeaRainbow Holding Corp, Sears Sites, Seat Pagine Gialle, Seesaa, Seiko Epson Corp, Seoul Newspaper, Sheknows (Evolve Media Corp.), Shopzilla.com Sites, Sky Sites, SkyCN, Skyrock, Slide Sites, Societe Generale, Softissimo, Sohu.com Inc., Sonic Solutions, Sonico Inc., Sony Online, SouFun.com Limited, Source Interlink Companies, Southwest Airlines Co., Sparkassen-Finanzgruppe, Spiegel Verlag, Spil Games, Sprint Nextel Corporation, Stage6 by DivX, Staples.com Sites, Stateless Systems, Steam (App), Sugar Inc - PopSugar Network, Sun Microsystems Inc., Superpages.com Network, Symantec, Systemax Inc., TAGSTORY.COM, TALEO.NET, TARINGA.NET, TBS Group, TD Bank Financial Group, TE3P.COM, TGBUS.COM, THEPIRATEBAY.ORG, TIEXUE.NET, TIMEANDDATE.COM, TIMNHANH.COM, TOGGLE.COM, TOM.COM Limited, TOPSHAREWARE.COM, TORRENTPORTAL.COM, TORRENTREACTOR.NET, TORRENTS.RU, TORRENTZ.COM, TQ Digital Entertainment, TUI Group, TV Guide Online Network, TWITTER.COM, Take 2 Interactive, Target Corporation, Tchibo, Technorati Media, Tele2 Sites, Telecom Italia, Telegraaf Media Groep, Telegraph Media Group, Telstra Corporation Limited, Terra - Telefonica, Tesco Stores, The Home Depot Inc., The Mozilla Organization, The Royal Bank Of Scotland, The Washington Post Company, The Weather Network Meteomedia, Thomson Reuters, Ticketmaster, Time Warner (Excl. Turner/WB), Times Internet Limited, Tiscali Sites, Toshiba Corp, Toyota, Toysrus Sites, Trader Media Group, Transcontinental Digital, Travelocity, Travian Games, Trend Micro Inc., Tribune Interactive, Trinity Mirror Group, Trovit Sites, Tudou Sites, Turner Digital, Twenga, UGO Entertainment, UOL, UPLOADED.TO, UPS Sites, UPTODOWN.COM, URBANDICTIONARY.COM, USAFIS.ORG, USEN Group, UUSEE.COM, UUU9.COM, Ubisoft Entertainment, Unilever, Unister GmbH, United Business Media, United Online Inc, United-Internet Sites, Universia, VATGIA.COM, VEOH.COM, VUI.VN, ValueClick Sites, Valve Corporation, Vector Inc., Veoh (App), Verizon Communications Corporation, Verlagsgruppe Georg von Holtzbrinck, VeryCD, Viacom Digital, Virgin Group, Virgin Media, VistaPrint Sites, Viva street Classifieds, Vivendi, Vocento, Vodafone Group, Volkswagen Group, Vuze Sites, WANMEI.COM, WARESEEKER.COM, WEBHATTI.COM, WIKIHOW.COM, WIKIMAPIA.ORG, WILEY.COM, WIND Telecomunicazioni, WORDREFERENCE.COM, WOYO.COM, Wal-Mart, Walgreen Co., Weather Channel The, WeatherBug Property, WebMD Health, WebMediaBrands Inc., Weblogs SL Sites, Weight Watchers International, Wells Fargo, WetterOnline GmbH, WhitePages, Wikia Sites, WildTangent Media, WordPress, World of Warcraft (App), WorldNow Sites, XANGA.COM, XAT.COM, XE.COM, XICI.NET, Xacti Corporation, Xinhua News Agency, Xunlei Networking, Y8.COM, YAMAHA, YAMEDIA INC., YE-SOFT.NET, YELP.COM, YOKA.COM*, YOURFILEHOST.COM, Yahoo, Yandex Sites, Yellow Pages Group, Yellowbook Network, Yomiuri Group, Youku Inc., Youthology Kids (Evolve Media Corp.), Yum! Brands Inc., ZHAOPIN.COM, ZHCW.COM, ZSHARE.NET, ZVue Networks, Zazzle, Ziff Davis, craigslist inc., eBuddy Sites, eMule (App), eNet, edaily, freenet AG, iEUROP Sites, iG, imeem, mIRC, spreadshirt
+
+
+Updates found, rewrite master file 'history_comscore_UV_properties.csv'
+
+WriteDataAnalyticsMySQL
+
+2008-01,-,AS,-,-,19.4,59964000
+2008-01,-,C,-,-,0.6,581000
+2008-01,-,EU,-,-,39.1,91036000
+2008-01,-,I,-,-,19.2,5138000
+2008-01,-,LA,-,-,31.1,18341000
+2008-01,-,MA,-,-,28.9,11549000
+2008-01,-,NA,-,-,33.5,61664000
+2008-01,-,W,-,-,29.4,242554000
+2008-02,-,AS,-,-,19.5,59948000
+2008-02,-,C,-,-,0.7,695000
+2008-02,-,EU,-,-,37.9,87975000
+2008-02,-,I,-,-,18.8,5065000
+2008-02,-,LA,-,-,34.2,20286000
+2008-02,-,MA,-,-,28.3,11327000
+2008-02,-,NA,-,-,33.2,61218000
+2008-02,-,W,-,-,29.3,240754000
+2008-03,-,AS,-,-,19.8,62380000
+2008-03,-,C,-,-,0.7,669000
+2008-03,-,EU,-,-,39.1,92283000
+2008-03,-,I,-,-,18.7,5200000
+2008-03,-,LA,-,-,40.4,24542000
+2008-03,-,MA,-,-,32.2,13337000
+2008-03,-,NA,-,-,34.0,63518000
+2008-03,-,W,-,-,30.5,256061000
+2008-04,-,AS,-,-,20.1,63963000
+2008-04,-,C,-,-,2.2,2288000
+2008-04,-,EU,-,-,39.3,93642000
+2008-04,-,I,-,-,17.9,5079000
+2008-04,-,LA,-,-,42.8,26347000
+2008-04,-,MA,-,-,32.3,13634000
+2008-04,-,NA,-,-,33.8,63827000
+2008-04,-,W,-,-,30.8,261414000
+2008-05,-,AS,-,-,19.7,63560000
+2008-05,-,C,-,-,0.9,994000
+2008-05,-,EU,-,-,39.7,95234000
+2008-05,-,I,-,-,18.2,5264000
+2008-05,-,LA,-,-,43.2,27002000
+2008-05,-,MA,-,-,32.9,14139000
+2008-05,-,NA,-,-,34.1,63186000
+2008-05,-,W,-,-,30.8,263120000
+2008-06,-,AS,-,-,20.1,65764000
+2008-06,-,C,-,-,0.9,958000
+2008-06,-,EU,-,-,37.0,89427000
+2008-06,-,I,-,-,20.4,5979000
+2008-06,-,LA,-,-,41.2,26099000
+2008-06,-,MA,-,-,27.3,11941000
+2008-06,-,NA,-,-,31.5,58272000
+2008-06,-,W,-,-,29.2,251502000
+2008-07,-,AS,-,-,17.2,65838000
+2008-07,-,C,-,-,0.1,232000
+2008-07,-,EU,-,-,32.4,86905000
+2008-07,-,I,-,-,19.2,5720000
+2008-07,-,LA,-,-,33.6,23220000
+2008-07,-,MA,-,-,24.7,11016000
+2008-07,-,NA,-,-,31.1,57347000
+2008-07,-,W,-,-,25.7,244326000
+2008-08,-,AS,-,-,17.2,66895000
+2008-08,-,C,-,-,0.8,1382000
+2008-08,-,EU,-,-,31.0,83931000
+2008-08,-,I,-,-,20.5,6217000
+2008-08,-,LA,-,-,38.2,26847000
+2008-08,-,MA,-,-,25.1,11371000
+2008-08,-,NA,-,-,32.4,59494000
+2008-08,-,W,-,-,25.9,248539000
+2008-09,-,AS,-,-,17.2,68291000
+2008-09,-,C,-,-,1.6,2735000
+2008-09,-,EU,-,-,35.9,98304000
+2008-09,-,I,-,-,19.6,6025000
+2008-09,-,LA,-,-,40.4,28886000
+2008-09,-,MA,-,-,27.2,12553000
+2008-09,-,NA,-,-,34.7,64075000
+2008-09,-,W,-,-,28.0,272109000
+2008-10,-,AS,-,-,16.3,65703000
+2008-10,-,C,-,-,1.2,2037000
+2008-10,-,EU,-,-,37.2,102955000
+2008-10,-,I,-,-,18.6,5799000
+2008-10,-,LA,-,-,38.6,27974000
+2008-10,-,MA,-,-,29.3,13790000
+2008-10,-,NA,-,-,36.0,66785000
+2008-10,-,W,-,-,28.2,277208000
+2008-11,-,AS,-,-,16.7,68371000
+2008-11,-,C,-,-,1.2,2185000
+2008-11,-,EU,-,-,38.0,106199000
+2008-11,-,I,-,-,19.9,6286000
+2008-11,-,LA,-,-,37.2,27416000
+2008-11,-,MA,-,-,28.9,13850000
+2008-11,-,NA,-,-,35.1,65133000
+2008-11,-,W,-,-,28.2,280969000
+2008-12,-,AS,-,-,15.9,66026000
+2008-12,-,C,-,-,1.3,2286000
+2008-12,-,EU,-,-,37.3,105318000
+2008-12,-,I,-,-,20.0,6421000
+2008-12,-,LA,-,-,30.4,22769000
+2008-12,-,MA,-,-,31.0,15103000
+2008-12,-,NA,-,-,34.5,63782000
+2008-12,-,W,-,-,27.1,272998000
+2009-01,-,AS,-,-,16.8,71203000
+2009-01,-,C,-,-,1.0,1825000
+2009-01,-,EU,-,-,39.4,112449000
+2009-01,-,I,-,-,20.8,6764000
+2009-01,-,LA,-,-,29.6,22558000
+2009-01,-,MA,-,-,31.5,15630000
+2009-01,-,NA,-,-,36.6,67971000
+2009-01,-,W,-,-,28.4,289811000
+2009-02,-,AS,-,-,16.4,70862000
+2009-02,-,C,-,-,1.1,2002000
+2009-02,-,EU,-,-,38.5,117828000
+2009-02,-,I,-,-,20.0,6587000
+2009-02,-,LA,-,-,33.9,27440000
+2009-02,-,MA,-,-,24.8,18765000
+2009-02,-,NA,-,-,35.5,65855000
+2009-02,-,W,-,-,27.9,300751000
+2009-03,-,AS,-,-,17.8,78009000
+2009-03,-,C,-,-,1.4,2752000
+2009-03,-,EU,-,-,40.9,126474000
+2009-03,-,I,-,-,21.0,7014000
+2009-03,-,LA,-,-,41.5,34229000
+2009-03,-,MA,-,-,28.2,21558000
+2009-03,-,NA,-,-,36.0,66878000
+2009-03,-,W,-,-,29.9,327148000
+2009-04,-,AS,-,-,17.0,76024000
+2009-04,-,C,-,-,1.6,3115000
+2009-04,-,EU,-,-,38.3,119578000
+2009-04,-,I,-,-,20.2,6807000
+2009-04,-,LA,-,-,39.8,33400000
+2009-04,-,MA,-,-,26.0,20133000
+2009-04,-,NA,-,-,37.8,70908000
+2009-04,-,W,-,-,28.9,320043000
+2009-05,-,AS,-,-,16.6,75595000
+2009-05,-,C,-,-,1.5,2936000
+2009-05,-,EU,-,-,37.6,118874000
+2009-05,-,I,-,-,21.0,7168000
+2009-05,-,LA,-,-,39.6,33870000
+2009-05,-,MA,-,-,24.9,19529000
+2009-05,-,NA,-,-,36.8,69387000
+2009-05,-,W,-,-,28.3,317255000
+2009-06,-,AS,-,-,16.5,76239000
+2009-06,-,C,-,-,2.2,4566000
+2009-06,-,EU,-,-,35.2,112270000
+2009-06,-,I,-,-,21.7,7522000
+2009-06,-,LA,-,-,38.3,33263000
+2009-06,-,MA,-,-,20.8,16489000
+2009-06,-,NA,-,-,34.4,64678000
+2009-06,-,W,-,-,26.7,302940000
+2009-07,-,AS,-,-,15.9,74990000
+2009-07,-,C,-,-,1.8,3839000
+2009-07,-,EU,-,-,33.7,108552000
+2009-07,-,I,-,-,22.6,7908000
+2009-07,-,LA,-,-,33.1,29298000
+2009-07,-,MA,-,-,20.6,16502000
+2009-07,-,NA,-,-,34.9,66142000
+2009-07,-,W,-,-,25.7,295484000
+2009-08,-,AS,-,-,16.2,77365000
+2009-08,-,C,-,-,1.6,3547000
+2009-08,-,EU,-,-,34.2,111190000
+2009-08,-,I,-,-,23.7,8398000
+2009-08,-,LA,-,-,36.8,33186000
+2009-08,-,MA,-,-,21.9,17816000
+2009-08,-,NA,-,-,35.6,68084000
+2009-08,-,W,-,-,26.4,307641000
+2009-09,-,AS,-,-,16.1,77948000
+2009-09,-,C,-,-,1.3,2882000
+2009-09,-,EU,-,-,37.4,123062000
+2009-09,-,I,-,-,23.7,8480000
+2009-09,-,LA,-,-,38.4,35264000
+2009-09,-,MA,-,-,22.0,18063000
+2009-09,-,NA,-,-,37.3,71661000
+2009-09,-,W,-,-,27.6,325998000
+2009-10,-,AS,-,-,17.0,83671000
+2009-10,-,C,-,-,1.6,3702000
+2009-10,-,EU,-,-,39.0,128647000
+2009-10,-,I,-,-,25.7,9287000
+2009-10,-,LA,-,-,39.0,36440000
+2009-10,-,MA,-,-,26.9,22353000
+2009-10,-,NA,-,-,38.2,73452000
+2009-10,-,W,-,-,29.0,344563000
+2009-11,-,AS,-,-,16.5,82379000
+2009-11,-,C,-,-,1.6,3718000
+2009-11,-,EU,-,-,39.0,129798000
+2009-11,-,I,-,-,25.8,9437000
+2009-11,-,LA,-,-,38.4,36495000
+2009-11,-,MA,-,-,26.2,22083000
+2009-11,-,NA,-,-,38.5,75050000
+2009-11,-,W,-,-,28.7,345805000
+2009-12,-,AS,-,-,16.7,84201000
+2009-12,-,C,-,-,1.5,3657000
+2009-12,-,EU,-,-,38.6,129763000
+2009-12,-,I,-,-,26.3,9696000
+2009-12,-,LA,-,-,32.5,31391000
+2009-12,-,MA,-,-,28.3,24058000
+2009-12,-,NA,-,-,39.1,77607000
+2009-12,-,W,-,-,28.4,347020000
+2010-01,-,AS,-,-,17.4,89049000
+2010-01,-,C,-,-,1.4,3351000
+2010-01,-,EU,-,-,40.3,136935000
+2010-01,-,I,-,-,27.4,10216000
+2010-01,-,LA,-,-,33.5,32834000
+2010-01,-,MA,-,-,27.2,23411000
+2010-01,-,NA,-,-,41.1,82489000
+2010-01,-,W,-,-,29.5,364719000
+2010-02,-,AS,-,-,15.3,72378000
+2010-02,-,C,-,-,1.2,2935000
+2010-02,-,EU,-,-,39.2,133807000
+2010-02,-,I,-,-,26.4,9865000
+2010-02,-,LA,-,-,36.7,36414000
+2010-02,-,MA,-,-,24.9,21680000
+2010-02,-,NA,-,-,39.9,80940000
+2010-02,-,W,-,-,28.7,345218000
+2010-03,-,AS,-,-,16.2,77678000
+2010-03,-,C,-,-,1.4,3443000
+2010-03,-,EU,-,-,40.8,140834000
+2010-03,-,I,-,-,27.7,10516000
+2010-03,-,LA,-,-,43.1,43574000
+2010-03,-,MA,-,-,28.3,24962000
+2010-03,-,NA,-,-,41.0,83696000
+2010-03,-,W,-,-,30.4,370744000
+2010-04,-,AS,-,-,15.9,77535000
+2010-04,-,C,-,-,1.6,4081000
+2010-04,-,EU,-,-,41.4,144087000
+2010-04,-,I,-,-,28.4,10923000
+2010-04,-,LA,-,-,43.2,44286000
+2010-04,-,MA,-,-,28.1,25146000
+2010-04,-,NA,-,-,40.9,83792000
+2010-04,-,W,-,-,30.4,374846000
+2010-05,-,AS,-,-,16.3,80502000
+2010-05,-,C,-,-,1.0,2700000
+2010-05,-,EU,-,-,43.0,150313000
+2010-05,-,I,-,-,30.1,11675000
+2010-05,-,LA,-,-,44.1,45790000
+2010-05,-,MA,-,-,29.2,26399000
+2010-05,-,NA,-,-,41.5,85929000
+2010-05,-,W,-,-,31.3,388932000
+2010-06,-,AS,-,-,16.1,80269000
+2010-06,-,C,-,-,1.0,2613000
+2010-06,-,EU,-,-,43.0,151013000
+2010-06,-,I,-,-,31.6,12373000
+2010-06,-,LA,-,-,41.8,43826000
+2010-06,-,MA,-,-,25.6,23611000
+2010-06,-,NA,-,-,38.9,80393000
+2010-06,-,W,-,-,30.2,379112000
+2010-07,-,AS,-,-,15.9,80260000
+2010-07,-,C,-,-,0.9,2536000
+2010-07,-,EU,-,-,39.3,138975000
+2010-07,-,I,-,-,32.4,12828000
+2010-07,-,LA,-,-,35.9,38111000
+2010-07,-,MA,-,-,24.4,22991000
+2010-07,-,NA,-,-,39.0,79888000
+2010-07,-,W,-,-,28.5,360225000
+2010-08,-,AS,-,-,16.2,83015000
+2010-08,-,C,-,-,1.0,2829000
+2010-08,-,EU,-,-,39.6,140560000
+2010-08,-,I,-,-,33.0,13187000
+2010-08,-,LA,-,-,39.9,42303000
+2010-08,-,MA,-,-,24.4,26220000
+2010-08,-,NA,-,-,39.7,81295000
+2010-08,-,W,-,-,29.0,373392000
+2010-09,-,AS,-,-,16.1,83119000
+2010-09,-,C,-,-,1.1,3181000
+2010-09,-,EU,-,-,43.5,155438000
+2010-09,-,I,-,-,33.3,13433000
+2010-09,-,LA,-,-,42.2,45265000
+2010-09,-,MA,-,-,26.5,28749000
+2010-09,-,NA,-,-,41.9,85606000
+2010-09,-,W,-,-,30.8,398178000
+2010-10,-,AS,-,-,16.0,83406000
+2010-10,-,C,-,-,1.1,3156000
+2010-10,-,EU,-,-,44.4,159384000
+2010-10,-,I,-,-,33.5,13674000
+2010-10,-,LA,-,-,41.6,45152000
+2010-10,-,MA,-,-,30.1,33049000
+2010-10,-,NA,-,-,42.7,87359000
+2010-10,-,W,-,-,31.3,408350000
+2010-11,-,AS,-,-,15.7,82736000
+2010-11,-,C,-,-,1.1,3108000
+2010-11,-,EU,-,-,45.8,165020000
+2010-11,-,I,-,-,33.1,13614000
+2010-11,-,LA,-,-,42.4,46617000
+2010-11,-,MA,-,-,27.6,30711000
+2010-11,-,NA,-,-,42.0,85732000
+2010-11,-,W,-,-,31.3,410816000
+2010-12,-,AS,-,-,15.2,80958000
+2010-12,-,C,-,-,1.2,3372000
+2010-12,-,EU,-,-,43.6,157837000
+2010-12,-,I,-,-,33.7,14004000
+2010-12,-,LA,-,-,34.8,38738000
+2010-12,-,MA,-,-,27.4,30852000
+2010-12,-,NA,-,-,42.7,87087000
+2010-12,-,W,-,-,29.9,395472000
+2011-01,-,AS,-,-,15.9,85593000
+2011-01,-,C,-,-,1.0,3048000
+2011-01,-,EU,-,-,46.4,168571000
+2011-01,-,I,-,-,35.9,15067000
+2011-01,-,LA,-,-,35.0,39446000
+2011-01,-,MA,-,-,28.2,32113000
+2011-01,-,NA,-,-,43.2,88233000
+2011-01,-,W,-,-,31.0,413957000
+2011-02,-,AS,-,-,14.2,77468000
+2011-02,-,C,-,-,0.9,2664000
+2011-02,-,EU,-,-,41.6,150775000
+2011-02,-,I,-,-,30.6,12960000
+2011-02,-,LA,-,-,36.3,41411000
+2011-02,-,MA,-,-,25.8,29722000
+2011-02,-,NA,-,-,39.4,80038000
+2011-02,-,W,-,-,28.3,379415000
+2011-03,-,AS,-,-,14.7,80997000
+2011-03,-,C,-,-,0.9,2854000
+2011-03,-,EU,-,-,43.1,156913000
+2011-03,-,I,-,-,30.4,13011000
+2011-03,-,LA,-,-,41.3,47583000
+2011-03,-,MA,-,-,28,32599000
+2011-03,-,NA,-,-,40.1,81918000
+2011-03,-,W,-,-,29.6,400011000
+2011-04,-,AS,-,-,14,77891000
+2011-04,-,C,-,-,0.9,2866000
+2011-04,-,EU,-,-,40.8,149211000
+2011-04,-,I,-,-,29.6,12801000
+2011-04,-,LA,-,-,38.7,45108000
+2011-04,-,MA,-,-,25.7,30272000
+2011-04,-,NA,-,-,38,78234000
+2011-04,-,W,-,-,27.9,380716000
+2011-05,-,AS,-,-,15.1,84666000
+2011-05,-,C,-,-,1.1,3492000
+2011-05,-,EU,-,-,44,161311000
+2011-05,-,I,-,-,31.4,13701000
+2011-05,-,LA,-,-,41.6,49095000
+2011-05,-,MA,-,-,28.5,33908000
+2011-05,-,NA,-,-,39.5,82081000
+2011-05,-,W,-,-,29.9,411061000
+2011-06,-,AS,-,-,15.2,86539000
+2011-06,-,C,-,-,1.2,3960000
+2011-06,-,EU,-,-,41.7,153883000
+2011-06,-,I,-,-,32.1,14163000
+2011-06,-,LA,-,-,39.6,47203000
+2011-06,-,MA,-,-,25.8,30985000
+2011-06,-,NA,-,-,39.1,80753000
+2011-06,-,W,-,-,28.9,399362000
+2011-07,-,AS,-,-,15.7,90378000
+2011-07,-,C,-,-,1.5,4814000
+2011-07,-,EU,-,-,39.6,146724000
+2011-07,-,I,-,-,33.6,14979000
+2011-07,-,LA,-,-,35.4,42700000
+2011-07,-,MA,-,-,24.6,29882000
+2011-07,-,NA,-,-,40.5,83859000
+2011-07,-,W,-,-,28.2,393543000
+2011-08,-,AS,-,-,16.7,97661000
+2011-08,-,C,-,-,1.6,5163000
+2011-08,-,EU,-,-,40.9,152109000
+2011-08,-,I,-,-,35.4,15944000
+2011-08,-,LA,-,-,43,52983000
+2011-08,-,MA,-,-,25.9,31749000
+2011-08,-,NA,-,-,42.4,88277000
+2011-08,-,W,-,-,30,422779000
+2011-09,-,AS,-,-,17.4,102341000
+2011-09,-,C,-,-,1.7,5709000
+2011-09,-,EU,-,-,44.6,166505000
+2011-09,-,I,-,-,35.6,16180000
+2011-09,-,LA,-,-,45.5,56613000
+2011-09,-,MA,-,-,29,35877000
+2011-09,-,NA,-,-,44.4,93193000
+2011-09,-,W,-,-,32,454529000
+2011-10,-,AS,-,-,17.9,106128000
+2011-10,-,C,-,-,2.2,7386000
+2011-10,-,EU,-,-,47,176918000
+2011-10,-,I,-,-,34.9,16019000
+2011-10,-,LA,-,-,45.3,57210000
+2011-10,-,MA,-,-,32.2,40119000
+2011-10,-,NA,-,-,45.4,96252000
+2011-10,-,W,-,-,33.3,476627000
+2011-11,-,AS,-,-,17.7,105296000
+2011-11,-,C,-,-,2.2,7355000
+2011-11,-,EU,-,-,47.1,178580000
+2011-11,-,I,-,-,35.4,16425000
+2011-11,-,LA,-,-,44.2,56544000
+2011-11,-,MA,-,-,31.5,39614000
+2011-11,-,NA,-,-,44.7,94689000
+2011-11,-,W,-,-,33,474723000
+2011-12,-,AS,-,-,17.4,103851000
+2011-12,-,C,-,-,2.7,8803000
+2011-12,-,EU,-,-,45.5,173496000
+2011-12,-,I,-,-,35.1,16425000
+2011-12,-,LA,-,-,35,45235000
+2011-12,-,MA,-,-,31.6,40068000
+2011-12,-,NA,-,-,44.8,94413000
+2011-12,-,W,-,-,31.7,457063000
+2012-01,-,AS,-,-,18.1,108186000
+2012-01,-,C,-,-,2.6,8766000
+2012-01,-,EU,-,-,48.1,184668000
+2012-01,-,I,-,-,36.4,17205000
+2012-01,-,LA,-,-,36.3,47386000
+2012-01,-,MA,-,-,32.9,42097000
+2012-01,-,NA,-,-,47.3,99821000
+2012-01,-,W,-,-,33.2,482157000
+2012-02,-,AS,-,-,17.9,105783000
+2012-02,-,C,-,-,2.4,7886000
+2012-02,-,EU,-,-,46.4,179385000
+2012-02,-,I,-,-,34.6,18771000
+2012-02,-,LA,-,-,39.6,51425000
+2012-02,-,MA,-,-,32.2,41056000
+2012-02,-,NA,-,-,46.3,98049000
+2012-02,-,W,-,-,32.8,475699000
+2012-03,-,AS,-,-,18.4,110909000
+2012-03,-,C,-,-,2.8,9249000
+2012-03,-,EU,-,-,46.2,180827000
+2012-03,-,I,-,-,34.5,19402000
+2012-03,-,LA,-,-,44.6,58584000
+2012-03,-,MA,-,-,32.4,42081000
+2012-03,-,NA,-,-,45.2,97001000
+2012-03,-,W,-,-,33.3,489402000
+WriteDataAnalyticsDygraph -> /a/wikistats/analytics
+
+
+Ready
+
+ParseArguments
+
+Arguments
+ -i /a/wikistats/csv/
+ -o /a/wikistats/analytics
+
+Input folder: /a/wikistats/csv/
+Output folder: /a/wikistats/analytics
+
+ReadStatisticsMonthly
+Read '/a/wikistats/csv//csv_wb/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wb/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wk/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wk/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wn/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wn/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wp/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wp/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wq/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wq/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_ws/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_ws/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wv/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wv/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wx/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wx/StatisticsUserActivitySpread.csv'
+FindLargestWikis
+Largest projects (most accumulated very active editors):
+wp,zz,wp,zz28,wp,en,wp,fr,wx,zz,wp,de,wx,commons,wx,zz28,wp,ru,wp,ja,wp,it,
+
+No data yet for large wiki wx,zz28 for 2012-04-> skip month 2012-04
+No data yet for large wiki wp,zz28 for 2012-04-> skip month 2012-04
+No data yet for large wiki wx,zz for 2012-04-> skip month 2012-04
+No data yet for large wiki wp,zz for 2012-04-> skip month 2012-04
+No data yet for large wiki wp,ru for 2012-05-> skip month 2012-05
+No data yet for large wiki wp,fr for 2012-05-> skip month 2012-05
+No data yet for large wiki wp,ja for 2012-05-> skip month 2012-05
+No data yet for large wiki wx,zz28 for 2012-05-> skip month 2012-05
+No data yet for large wiki wp,zz28 for 2012-05-> skip month 2012-05
+No data yet for large wiki wx,zz for 2012-05-> skip month 2012-05
+No data yet for large wiki wp,it for 2012-05-> skip month 2012-05
+No data yet for large wiki wp,zz for 2012-05-> skip month 2012-05
+No data yet for large wiki wx,commons for 2012-05-> skip month 2012-05
+No data yet for large wiki wp,en for 2012-05-> skip month 2012-05
+No data yet for large wiki wp,de for 2012-05-> skip month 2012-05
+WriteMonthlyData
+Editor data missing for wb,ba,2004-10
+Editor data missing for wb,ba,2004-11
+Editor data missing for wb,ba,2004-12
+Editor data missing for wb,ba,2005-01
+Editor data missing for wb,ba,2005-02
+Editor data missing for wb,ba,2005-03
+Editor data missing for wb,ba,2005-04
+Editor data missing for wb,ba,2005-05
+Editor data missing for wb,ba,2005-06
+Editor data missing for wb,ba,2005-07
+Editor data missing for wb,ba,2005-08
+Editor data missing for wb,ba,2005-09
+Editor data missing for wb,ba,2005-10
+Editor data missing for wb,ba,2005-11
+Editor data missing for wb,ba,2005-12
+Editor data missing for wb,ba,2006-01
+Editor data missing for wb,ba,2006-02
+Editor data missing for wb,ba,2006-03
+Editor data missing for wb,ba,2006-04
+Editor data missing for wb,ba,2006-05
+Editor data missing for wb,ba,2006-06
+Editor data missing for wb,ba,2006-07
+Editor data missing for wb,ba,2006-08
+Editor data missing for wb,ba,2006-09
+Editor data missing for wb,ba,2006-10
+Editor data missing for wb,ba,2006-11
+Editor data missing for wb,ba,2006-12
+Editor data missing for wb,ba,2007-01
+Editor data missing for wb,ba,2007-02
+Editor data missing for wb,ba,2007-03
+Editor data missing for wb,ba,2007-04
+Editor data missing for wb,ba,2007-05
+Editor data missing for wb,ba,2007-06
+Editor data missing for wb,ba,2007-07
+Editor data missing for wb,ba,2007-08
+Editor data missing for wb,ba,2007-09
+Editor data missing for wb,ba,2007-10
+Editor data missing for wb,ba,2007-11
+Editor data missing for wb,ba,2007-12
+Editor data missing for wb,ba,2008-01
+Editor data missing for wb,ba,2008-02
+Editor data missing for wb,ba,2008-03
+Editor data missing for wb,ba,2008-04
+Editor data missing for wb,ba,2008-05
+Editor data missing for wb,ba,2008-06
+Editor data missing for wb,ba,2008-07
+Editor data missing for wb,ba,2008-08
+Editor data missing for wb,ba,2008-09
+Editor data missing for wb,ba,2008-10
+Editor data missing for wb,ba,2008-11
+Editor data missing for wq,cr,2005-01
+Editor data missing for wq,cr,2005-02
+Editor data missing for wq,cr,2005-03
+Editor data missing for wq,cr,2005-04
+Editor data missing for wq,cr,2005-05
+Editor data missing for wq,cr,2005-06
+Editor data missing for wq,cr,2005-07
+Editor data missing for wq,cr,2005-08
+Editor data missing for wq,cr,2005-09
+Editor data missing for wq,cr,2005-10
+Editor data missing for wq,cr,2005-11
+Editor data missing for wq,cr,2005-12
+Editor data missing for wq,cr,2006-01
+Editor data missing for wq,cr,2006-02
+Editor data missing for wq,cr,2006-03
+Editor data missing for wq,cr,2006-04
+Editor data missing for wq,cr,2006-05
+Editor data missing for wq,cr,2006-06
+Editor data missing for wq,cr,2006-07
+Editor data missing for wq,cr,2006-08
+Editor data missing for wq,cr,2006-09
+Editor data missing for wq,cr,2006-10
+Editor data missing for wq,cr,2006-11
+Editor data missing for wq,cr,2006-12
+Editor data missing for wq,cr,2007-01
+Editor data missing for wq,cr,2007-02
+Editor data missing for wq,cr,2007-03
+Editor data missing for wq,cr,2007-04
+Editor data missing for wq,cr,2007-05
+Editor data missing for wq,cr,2007-06
+Editor data missing for wq,cr,2007-07
+Editor data missing for wq,cr,2007-08
+Editor data missing for wq,cr,2007-09
+Editor data missing for wq,cr,2007-10
+Editor data missing for wq,cr,2007-11
+Editor data missing for wq,cr,2007-12
+Editor data missing for wq,cr,2008-01
+Editor data missing for wq,cr,2008-02
+Editor data missing for wq,cr,2008-03
+Editor data missing for wq,cr,2008-04
+Editor data missing for wq,cr,2008-05
+Editor data missing for wq,cr,2008-06
+Editor data missing for wq,cr,2008-07
+Editor data missing for wq,cr,2008-08
+Editor data missing for wq,cr,2008-09
+Editor data missing for wq,cr,2008-10
+Editor data missing for wq,cr,2008-11
+
+Ready
+
+
+Script started at Mon, Jun 4, 2012 23:03
+Arguments: -i | /a/wikistats/csv/ | -o | /a/wikistats/analytics | -m | 2012-04
+
+================================================================================
+
+
+Arguments
+ -i /a/wikistats/csv/
+ -m 2012-04
+ -o /a/wikistats/analytics
+
+Input folder: /a/wikistats/csv/
+Output folder: /a/wikistats/analytics
+Period 2009-04 - 2012-04
+
+
+Write trend data up till 2012/04
+
+Compare with previous month: 2012/03, previous year: 2011/04
+
+Read '/a/wikistats/csv//csv_wb/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wb/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wk/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wk/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wn/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wn/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wp/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wp/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wq/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wq/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_ws/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_ws/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wv/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wv/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wx/StatisticsMonthly.csv'
+Read '/a/wikistats/csv//csv_wx/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats/csv//csv_wx/StatisticsPerBinariesExtension.csv'
+Write file '/a/wikistats/analytics/StatisticsMonthly_2012_04.csv'
+project wp en
+project wx commons
+project wp fr
+project wp de
+project wp ru
+project wp ja
+project wp es
+project wp it
+project wp zh
+project wp pt
+project wp pl
+project wp nl
+project wp sv
+project wp uk
+project wp he
+project wp ko
+project wp ar
+project wp cs
+project wp hu
+project wp fa
+project wp fi
+project wp no
+project wp id
+project wp tr
+project wx meta
+
+Output written to /a/wikistats/analytics/StatisticsMonthly_2012_04.csv
+
+Write copy of /a/wikistats/analytics/StatisticsMonthly_2012_04.csv to /a/wikistats/analytics/wikilytics_in_wikistats_core_metrics.csv
+
+================================================================================
+
+
+
+Prog:
+Path: AnalyticsPrepWikiCountsOutputCore.pl
+Host: stat1 (production)
+
+Args:
+
+ -i
+ /a/wikistats/csv/
+ -o
+ /a/wikistats/analytics
+ -m
+ 2012-04
+OS: linux
+Perl: 5.010001000
+Perl: /usr/bin/perl
+EzLib: 14
+
+Libs:
+ /home/ezachte/lib
+ /etc/perl
+ /usr/local/lib/perl/5.10.1
+ /usr/local/share/perl/5.10.1
+ /usr/lib/perl5
+ /usr/share/perl5
+ /usr/lib/perl/5.10
+ /usr/share/perl/5.10
+ /usr/local/lib/site_perl
+ .
+
+Own modules (d/m/y h:m):
+ 17/09/2011 1:04 /home/ezachte/lib/EzLib.pm
+
+
+
+
+
+================================================================================
+================================================================================
+

Added: trunk/wikistats/MetricsReporting/analytics/analytics_new.sh
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/analytics_new.sh (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/analytics_new.sh 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,5 @@
+clear
+cd /a/wikistats/analytics
+# rm *test*.csv
+# mysql --user=root --password=changerootwhenpriyankareturns < analytics_create_and_load_from_csv.txt > mysql_log.txt
+ mysql -u analytics -h project2.wikimedia.org -preport < analytics_create_and_load_from_csv.txt > mysql_log.txt

Added: trunk/wikistats/MetricsReporting/analytics/analytics_refresh_from_csv.txt
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/analytics_refresh_from_csv.txt (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/analytics_refresh_from_csv.txt 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,55 @@
+USE `analytics` ;
+
+TRUNCATE TABLE comscore ;
+LOAD DATA LOCAL INFILE 'analytics_in_comscore.csv'
+ INTO TABLE comscore
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (@date,country_code,region_code,web_property,project_code,reach,visitors)
+ SET date = last_day (str_to_date(concat (@date,'-01'),'%Y-%m-%d')) ;
+
+TRUNCATE TABLE comscore_regions ;
+LOAD DATA LOCAL INFILE 'analytics_in_comscore_regions.csv'
+ INTO TABLE comscore_regions
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (report_language,region_code,region_name) ;
+
+TRUNCATE TABLE wikistats ;
+LOAD DATA LOCAL INFILE 'analytics_in_wikistats.csv'
+ INTO TABLE wikistats
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (project_code,language_code,@date,editors_all_time,editors_new,editors_ge_5,editors_ge_25,editors_ge_100,articles,articles_new_per_day,articles_over_bytes_500,articles_over_bytes_2000,edits_per_article,bytes_per_article,edits,size_in_bytes,size_in_words,links_internal,links_interwiki,links_image,links_external,redirects)
+ SET date = last_day (str_to_date(concat (@date,'-01'),'%Y-%m-%d')) ;
+
+TRUNCATE TABLE page_views ;
+LOAD DATA LOCAL INFILE 'analytics_in_page_views.csv'
+ INTO TABLE page_views
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (project_code,language_code,@date,views_non_mobile_raw,views_mobile_raw,views_non_mobile_normalized,views_mobile_normalized,views_raw,views_normalized)
+ SET date = last_day (str_to_date(concat (@date,'-01'),'%Y-%m-%d')) ;
+
+TRUNCATE TABLE language_names ;
+LOAD DATA LOCAL INFILE 'analytics_in_language_names.csv'
+ INTO TABLE language_names
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (report_language,language_code,language_name) ;
+
+TRUNCATE TABLE binaries ;
+LOAD DATA LOCAL INFILE 'analytics_in_binaries.csv'
+ INTO TABLE binaries
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (project_code,language_code,@date,extension,binaries)
+ SET date = last_day (str_to_date(concat (@date,'-01'),'%Y-%m-%d')) ;
+
+TRUNCATE TABLE offline ;
+LOAD DATA LOCAL INFILE 'analytics_in_offline.csv'
+ INTO TABLE offline
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ (@date, readers)
+ SET date = last_day (str_to_date(concat (@date,'-01'),'%Y-%m-%d')) ;

Added: trunk/wikistats/MetricsReporting/analytics/analytics_test.sh
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/analytics_test.sh (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/analytics_test.sh 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,4 @@
+clear
+cd /a/wikistats/analytics
+
+mysql -u analytics -h project2.wikimedia.org -preport < analytics_test_query.txt

Added: trunk/wikistats/MetricsReporting/analytics/analytics_test_query.txt
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/analytics_test_query.txt (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/analytics_test_query.txt 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,7 @@
+USE `analytics` ;
+
+
+DESCRIBE page_views ;
+
+SELECT date, project_code,language_code FROM page_views
+WHERE project_code = 'wp' AND language_code = 'en' ;

Added: trunk/wikistats/MetricsReporting/analytics/analytics_upd.sh
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/analytics_upd.sh (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/analytics_upd.sh 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,6 @@
+clear
+cd /a/wikistats/analytics
+
+./analytics_generate_csv_files.sh
+
+# mysql -u analytics -h project2.wikimedia.org -preport < analytics_refresh_from_csv.txt

Added: trunk/wikistats/MetricsReporting/analytics/analytics_upd_log.txt
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/analytics_upd_log.txt (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/analytics_upd_log.txt 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,903 @@
+[.H[.2J[.H[.2JInput folder: /a/wikistats/
+Output folder: /a/analytics/
+
+Write file '/a/analytics//analytics_in_binaries.csv'
+Read '/a/wikistats//csv_wb/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats//csv_wk/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats//csv_wn/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats//csv_wp/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats//csv_wq/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats//csv_ws/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats//csv_wv/StatisticsPerBinariesExtension.csv'
+Read '/a/wikistats//csv_wx/StatisticsPerBinariesExtension.csv'
+
+
+Ready
+
+Mode is replace (specify '-r' for replace)
+
+Directories:
+Input (new comScore data): '/a/analytics/comscore'
+Master files (full history): '/a/analytics'
+Output (database feed): '/a/analytics'
+
+ReadMasterComscoreDataReachPerRegion
+
+Regions found: Europe,North America,Latin America,World-Wide,Middle East - Africa,India,Asia Pacific,China
+UpdateMasterFileFromRecentComscoreData
+
+White list: Europe,North America,Latin America,World-Wide,Middle East - Africa,India,Asia Pacific,China
+
+Latest comscore master file is 25 days old: 'history_comscore_reach_regions.csv'
+Latest comscore update file is 0 days old: 'Multi-Country Media Trend, % reach by region_(Jun 10 - Aug 11).csv'
+Data found: 2010-06,LA = 41.8
+Data found: 2010-07,LA = 35.9
+Data found: 2010-08,LA = 39.9
+Data found: 2010-09,LA = 42.2
+Data found: 2010-10,LA = 41.6
+Data found: 2010-11,LA = 42.4
+Data found: 2010-12,LA = 34.8
+Data found: 2011-01,LA = 35
+Data found: 2011-02,LA = 36.3
+Data found: 2011-03,LA = 41.3
+Data found: 2011-04,LA = 38.7
+Data found: 2011-05,LA = 41.6
+Data found: 2011-06,LA = 39.6
+Data found: 2011-07,LA = 35.4
+Data found: 2011-08,LA = 43
+Data found: 2010-06,NA = 38.9
+Data found: 2010-07,NA = 39
+Data found: 2010-08,NA = 39.7
+Data found: 2010-09,NA = 41.9
+Data found: 2010-10,NA = 42.7
+Data found: 2010-11,NA = 42
+Data found: 2010-12,NA = 42.7
+Data found: 2011-01,NA = 43.2
+Data found: 2011-02,NA = 39.4
+Data found: 2011-03,NA = 40.1
+Data found: 2011-04,NA = 38
+Data found: 2011-05,NA = 39.5
+Data found: 2011-06,NA = 39.1
+Data found: 2011-07,NA = 40.5
+Data found: 2011-08,NA = 42.4
+Data found: 2010-06,EU = 43
+Data found: 2010-07,EU = 39.3
+Data found: 2010-08,EU = 39.6
+Data found: 2010-09,EU = 43.5
+Data found: 2010-10,EU = 44.4
+Data found: 2010-11,EU = 45.8
+Data found: 2010-12,EU = 43.6
+Data found: 2011-01,EU = 46.4
+Data found: 2011-02,EU = 41.6
+Data found: 2011-03,EU = 43.1
+Data found: 2011-04,EU = 40.8
+Data found: 2011-05,EU = 44
+Data found: 2011-06,EU = 41.7
+Data found: 2011-07,EU = 39.6
+Data found: 2011-08,EU = 40.9
+Data found: 2010-06,I = 31.6
+Data found: 2010-07,I = 32.4
+Data found: 2010-08,I = 33
+Data found: 2010-09,I = 33.3
+Data found: 2010-10,I = 33.5
+Data found: 2010-11,I = 33.1
+Data found: 2010-12,I = 33.7
+Data found: 2011-01,I = 35.9
+Data found: 2011-02,I = 30.6
+Data found: 2011-03,I = 30.4
+Data found: 2011-04,I = 29.6
+Data found: 2011-05,I = 31.4
+Data found: 2011-06,I = 32.1
+Data found: 2011-07,I = 33.6
+Data found: 2011-08,I = 35.4
+Data found: 2010-06,W = 30.2
+Data found: 2010-07,W = 28.5
+Data found: 2010-08,W = 29
+Data found: 2010-09,W = 30.8
+Data found: 2010-10,W = 31.3
+Data found: 2010-11,W = 31.3
+Data found: 2010-12,W = 29.9
+Data found: 2011-01,W = 31
+Data found: 2011-02,W = 28.3
+Data found: 2011-03,W = 29.6
+Data found: 2011-04,W = 27.9
+Data found: 2011-05,W = 29.9
+Data found: 2011-06,W = 28.9
+Data found: 2011-07,W = 28.2
+Data found: 2011-08,W = 30
+Data found: 2010-06,MA = 25.6
+Data found: 2010-07,MA = 24.4
+Data found: 2010-08,MA = 24.4
+Data found: 2010-09,MA = 26.5
+Data found: 2010-10,MA = 30.1
+Data found: 2010-11,MA = 27.6
+Data found: 2010-12,MA = 27.4
+Data found: 2011-01,MA = 28.2
+Data found: 2011-02,MA = 25.8
+Data found: 2011-03,MA = 28
+Data found: 2011-04,MA = 25.7
+Data found: 2011-05,MA = 28.5
+Data found: 2011-06,MA = 25.8
+Data found: 2011-07,MA = 24.6
+Data found: 2011-08,MA = 25.9
+Data found: 2010-06,AS = 16.1
+Data found: 2010-07,AS = 15.9
+Data found: 2010-08,AS = 16.2
+Data found: 2010-09,AS = 16.1
+Data found: 2010-10,AS = 16
+Data found: 2010-11,AS = 15.7
+Data found: 2010-12,AS = 15.2
+Data found: 2011-01,AS = 15.9
+Data found: 2011-02,AS = 14.2
+Data found: 2011-03,AS = 14.7
+Data found: 2011-04,AS = 14
+Data found: 2011-05,AS = 15.1
+Data found: 2011-06,AS = 15.2
+Data found: 2011-07,AS = 15.7
+Data found: 2011-08,AS = 16.7
+Data found: 2010-06,C = 1
+Data found: 2010-07,C = 0.9
+Data found: 2010-08,C = 1
+Data found: 2010-09,C = 1.1
+Data found: 2010-10,C = 1.1
+Data found: 2010-11,C = 1.1
+Data found: 2010-12,C = 1.2
+Data found: 2011-01,C = 1
+Data found: 2011-02,C = 0.9
+Data found: 2011-03,C = 0.9
+Data found: 2011-04,C = 0.9
+Data found: 2011-05,C = 1.1
+Data found: 2011-06,C = 1.2
+Data found: 2011-07,C = 1.5
+Data found: 2011-08,C = 1.6
+
+Updates found, rewrite master file 'history_comscore_reach_regions.csv'
+
+ReadMasterComscoreDataVisitorsPerRegion
+
+Regions found: World-Wide,Europe,Asia Pacific,North America,Latin America,Middle East - Africa,India,China
+UpdateMasterFileFromRecentComscoreData
+
+White list: World-Wide,Europe,Asia Pacific,North America,Latin America,Middle East - Africa,India,China
+
+Latest comscore master file is 25 days old: 'history_comscore_UV_regions.csv'
+Latest comscore update file is 0 days old: 'Multi-Country Media Trend, UVs by region_(Jun 10 - Aug 11).csv'
+Data found: 2010-06,W = 379112000
+Data found: 2010-07,W = 360225000
+Data found: 2010-08,W = 373392000
+Data found: 2010-09,W = 398178000
+Data found: 2010-10,W = 408350000
+Data found: 2010-11,W = 410816000
+Data found: 2010-12,W = 395472000
+Data found: 2011-01,W = 413957000
+Data found: 2011-02,W = 379415000
+Data found: 2011-03,W = 400011000
+Data found: 2011-04,W = 380716000
+Data found: 2011-05,W = 411061000
+Data found: 2011-06,W = 399362000
+Data found: 2011-07,W = 393543000
+Data found: 2011-08,W = 422779000
+Data found: 2010-06,EU = 151013000
+Data found: 2010-07,EU = 138975000
+Data found: 2010-08,EU = 140560000
+Data found: 2010-09,EU = 155438000
+Data found: 2010-10,EU = 159384000
+Data found: 2010-11,EU = 165020000
+Data found: 2010-12,EU = 157837000
+Data found: 2011-01,EU = 168571000
+Data found: 2011-02,EU = 150775000
+Data found: 2011-03,EU = 156913000
+Data found: 2011-04,EU = 149211000
+Data found: 2011-05,EU = 161311000
+Data found: 2011-06,EU = 153883000
+Data found: 2011-07,EU = 146724000
+Data found: 2011-08,EU = 152109000
+Data found: 2010-06,AS = 80269000
+Data found: 2010-07,AS = 80260000
+Data found: 2010-08,AS = 83015000
+Data found: 2010-09,AS = 83119000
+Data found: 2010-10,AS = 83406000
+Data found: 2010-11,AS = 82736000
+Data found: 2010-12,AS = 80958000
+Data found: 2011-01,AS = 85593000
+Data found: 2011-02,AS = 77468000
+Data found: 2011-03,AS = 80997000
+Data found: 2011-04,AS = 77891000
+Data found: 2011-05,AS = 84666000
+Data found: 2011-06,AS = 86539000
+Data found: 2011-07,AS = 90378000
+Data found: 2011-08,AS = 97661000
+Data found: 2010-06,NA = 80393000
+Data found: 2010-07,NA = 79888000
+Data found: 2010-08,NA = 81295000
+Data found: 2010-09,NA = 85606000
+Data found: 2010-10,NA = 87359000
+Data found: 2010-11,NA = 85732000
+Data found: 2010-12,NA = 87087000
+Data found: 2011-01,NA = 88233000
+Data found: 2011-02,NA = 80038000
+Data found: 2011-03,NA = 81918000
+Data found: 2011-04,NA = 78234000
+Data found: 2011-05,NA = 82081000
+Data found: 2011-06,NA = 80753000
+Data found: 2011-07,NA = 83859000
+Data found: 2011-08,NA = 88277000
+Data found: 2010-06,LA = 43826000
+Data found: 2010-07,LA = 38111000
+Data found: 2010-08,LA = 42303000
+Data found: 2010-09,LA = 45265000
+Data found: 2010-10,LA = 45152000
+Data found: 2010-11,LA = 46617000
+Data found: 2010-12,LA = 38738000
+Data found: 2011-01,LA = 39446000
+Data found: 2011-02,LA = 41411000
+Data found: 2011-03,LA = 47583000
+Data found: 2011-04,LA = 45108000
+Data found: 2011-05,LA = 49095000
+Data found: 2011-06,LA = 47203000
+Data found: 2011-07,LA = 42700000
+Data found: 2011-08,LA = 52983000
+Data found: 2010-06,MA = 23611000
+Data found: 2010-07,MA = 22991000
+Data found: 2010-08,MA = 26220000
+Data found: 2010-09,MA = 28749000
+Data found: 2010-10,MA = 33049000
+Data found: 2010-11,MA = 30711000
+Data found: 2010-12,MA = 30852000
+Data found: 2011-01,MA = 32113000
+Data found: 2011-02,MA = 29722000
+Data found: 2011-03,MA = 32599000
+Data found: 2011-04,MA = 30272000
+Data found: 2011-05,MA = 33908000
+Data found: 2011-06,MA = 30985000
+Data found: 2011-07,MA = 29882000
+Data found: 2011-08,MA = 31749000
+Data found: 2010-06,I = 12373000
+Data found: 2010-07,I = 12828000
+Data found: 2010-08,I = 13187000
+Data found: 2010-09,I = 13433000
+Data found: 2010-10,I = 13674000
+Data found: 2010-11,I = 13614000
+Data found: 2010-12,I = 14004000
+Data found: 2011-01,I = 15067000
+Data found: 2011-02,I = 12960000
+Data found: 2011-03,I = 13011000
+Data found: 2011-04,I = 12801000
+Data found: 2011-05,I = 13701000
+Data found: 2011-06,I = 14163000
+Data found: 2011-07,I = 14979000
+Data found: 2011-08,I = 15944000
+Data found: 2010-06,C = 2613000
+Data found: 2010-07,C = 2536000
+Data found: 2010-08,C = 2829000
+Data found: 2010-09,C = 3181000
+Data found: 2010-10,C = 3156000
+Data found: 2010-11,C = 3108000
+Data found: 2010-12,C = 3372000
+Data found: 2011-01,C = 3048000
+Data found: 2011-02,C = 2664000
+Data found: 2011-03,C = 2854000
+Data found: 2011-04,C = 2866000
+Data found: 2011-05,C = 3492000
+Data found: 2011-06,C = 3960000
+Data found: 2011-07,C = 4814000
+Data found: 2011-08,C = 5163000
+
+Entities ignored:
+United States
+
+
+Updates found, rewrite master file 'history_comscore_UV_regions.csv'
+
+ReadMasterComscoreDataVisitorsPerProperty
+
+UpdateMasterFileFromRecentComscoreData
+
+White list: Total Internet,Google,Microsoft,Facebook,Wikimedia,Yahoo!,Amazon,Apple,AOL,Ask Network,eBay,CBS,Tencent,Glam Media,Baidu
+
+Latest comscore master file is 25 days old: 'history_comscore_UV_properties.csv'
+Latest comscore update file is 34 days old: 'Top 1000 poperties, UV trend_(May 10 - Jul 11).csv'
+Data found: 2010-05,Total Internet = 1244104000
+Data found: 2010-06,Total Internet = 1253406000
+Data found: 2010-07,Total Internet = 1261762000
+Data found: 2010-08,Total Internet = 1285378000
+Data found: 2010-09,Total Internet = 1294784000
+Data found: 2010-10,Total Internet = 1304632000
+Data found: 2010-11,Total Internet = 1313926000
+Data found: 2010-12,Total Internet = 1323535000
+Data found: 2011-01,Total Internet = 1333403000
+Data found: 2011-02,Total Internet = 1339496000
+Data found: 2011-03,Total Internet = 1350539000
+Data found: 2011-04,Total Internet = 1362369000
+Data found: 2011-05,Total Internet = 1373976000
+Data found: 2011-06,Total Internet = 1383098000
+Data found: 2011-07,Total Internet = 1394031000
+Data found: 2010-05,Google = 931583000
+Data found: 2010-06,Google = 942199000
+Data found: 2010-07,Google = 952484000
+Data found: 2010-08,Google = 971045000
+Data found: 2010-09,Google = 976431000
+Data found: 2010-10,Google = 976693000
+Data found: 2010-11,Google = 970109000
+Data found: 2010-12,Google = 972486000
+Data found: 2011-01,Google = 985289000
+Data found: 2011-02,Google = 976374000
+Data found: 2011-03,Google = 996706000
+Data found: 2011-04,Google = 990970000
+Data found: 2011-05,Google = 1009699000
+Data found: 2011-06,Google = 1023643000
+Data found: 2011-07,Google = 1031584000
+Data found: 2010-05,Microsoft = 789035000
+Data found: 2010-06,Microsoft = 799898000
+Data found: 2010-07,Microsoft = 800636000
+Data found: 2010-08,Microsoft = 814562000
+Data found: 2010-09,Microsoft = 827496000
+Data found: 2010-10,Microsoft = 840677000
+Data found: 2010-11,Microsoft = 869373000
+Data found: 2010-12,Microsoft = 867914000
+Data found: 2011-01,Microsoft = 889163000
+Data found: 2011-02,Microsoft = 837039000
+Data found: 2011-03,Microsoft = 895813000
+Data found: 2011-04,Microsoft = 892893000
+Data found: 2011-05,Microsoft = 905454000
+Data found: 2011-06,Microsoft = 882818000
+Data found: 2011-07,Microsoft = 898728000
+Data found: 2010-05,Facebook = 548134000
+Data found: 2010-06,Facebook = 550939000
+Data found: 2010-07,Facebook = 571480000
+Data found: 2010-08,Facebook = 598084000
+Data found: 2010-09,Facebook = 620269000
+Data found: 2010-10,Facebook = 633548000
+Data found: 2010-11,Facebook = 647482000
+Data found: 2010-12,Facebook = 662420000
+Data found: 2011-01,Facebook = 675361000
+Data found: 2011-02,Facebook = 676733000
+Data found: 2011-03,Facebook = 692998000
+Data found: 2011-04,Facebook = 697970000
+Data found: 2011-05,Facebook = 713647000
+Data found: 2011-06,Facebook = 734240000
+Data found: 2011-07,Facebook = 736465000
+Data found: 2010-05,Wikimedia = 388932000
+Data found: 2010-06,Wikimedia = 379112000
+Data found: 2010-07,Wikimedia = 360225000
+Data found: 2010-08,Wikimedia = 373392000
+Data found: 2010-09,Wikimedia = 398178000
+Data found: 2010-10,Wikimedia = 408350000
+Data found: 2010-11,Wikimedia = 410816000
+Data found: 2010-12,Wikimedia = 395472000
+Data found: 2011-01,Wikimedia = 413957000
+Data found: 2011-02,Wikimedia = 379415000
+Data found: 2011-03,Wikimedia = 400011000
+Data found: 2011-04,Wikimedia = 380716000
+Data found: 2011-05,Wikimedia = 411061000
+Data found: 2011-06,Wikimedia = 399362000
+Data found: 2011-07,Wikimedia = 393543000
+Data found: 2010-05,Amazon = 228083000
+Data found: 2010-06,Amazon = 221785000
+Data found: 2010-07,Amazon = 230554000
+Data found: 2010-08,Amazon = 237156000
+Data found: 2010-09,Amazon = 234334000
+Data found: 2010-10,Amazon = 245082000
+Data found: 2010-11,Amazon = 252698000
+Data found: 2010-12,Amazon = 263450000
+Data found: 2011-01,Amazon = 258169000
+Data found: 2011-02,Amazon = 233765000
+Data found: 2011-03,Amazon = 271264000
+Data found: 2011-04,Amazon = 268611000
+Data found: 2011-05,Amazon = 284321000
+Data found: 2011-06,Amazon = 282233000
+Data found: 2011-07,Amazon = 288766000
+Data found: 2010-05,Apple = 213166000
+Data found: 2010-06,Apple = 213671000
+Data found: 2010-07,Apple = 212905000
+Data found: 2010-08,Apple = 219498000
+Data found: 2010-09,Apple = 226129000
+Data found: 2010-10,Apple = 227768000
+Data found: 2010-11,Apple = 228285000
+Data found: 2010-12,Apple = 240557000
+Data found: 2011-01,Apple = 236347000
+Data found: 2011-02,Apple = 221271000
+Data found: 2011-03,Apple = 234944000
+Data found: 2011-04,Apple = 228366000
+Data found: 2011-05,Apple = 244182000
+Data found: 2011-06,Apple = 246889000
+Data found: 2011-07,Apple = 256743000
+Data found: 2010-05,Tencent = 175771000
+Data found: 2010-06,Tencent = 177789000
+Data found: 2010-07,Tencent = 182572000
+Data found: 2010-08,Tencent = 191021000
+Data found: 2010-09,Tencent = 197617000
+Data found: 2010-10,Tencent = 204266000
+Data found: 2010-11,Tencent = 211656000
+Data found: 2010-12,Tencent = 215211000
+Data found: 2011-01,Tencent = 208512000
+Data found: 2011-02,Tencent = 207793000
+Data found: 2011-03,Tencent = 214254000
+Data found: 2011-04,Tencent = 216190000
+Data found: 2011-05,Tencent = 220707000
+Data found: 2011-06,Tencent = 229175000
+Data found: 2011-07,Tencent = 237849000
+Data found: 2010-05,AOL = 266787000
+Data found: 2010-06,AOL = 255895000
+Data found: 2010-07,AOL = 248233000
+Data found: 2010-08,AOL = 221158000
+Data found: 2010-09,AOL = 218426000
+Data found: 2010-10,AOL = 232439000
+Data found: 2010-11,AOL = 239205000
+Data found: 2010-12,AOL = 239550000
+Data found: 2011-01,AOL = 243275000
+Data found: 2011-02,AOL = 229394000
+Data found: 2011-03,AOL = 247697000
+Data found: 2011-04,AOL = 237258000
+Data found: 2011-05,AOL = 238860000
+Data found: 2011-06,AOL = 231564000
+Data found: 2011-07,AOL = 227626000
+Data found: 2010-05,Baidu = 180965000
+Data found: 2010-06,Baidu = 174672000
+Data found: 2010-07,Baidu = 179920000
+Data found: 2010-08,Baidu = 181984000
+Data found: 2010-09,Baidu = 184728000
+Data found: 2010-10,Baidu = 185318000
+Data found: 2010-11,Baidu = 184703000
+Data found: 2010-12,Baidu = 185005000
+Data found: 2011-01,Baidu = 190956000
+Data found: 2011-02,Baidu = 192845000
+Data found: 2011-03,Baidu = 202995000
+Data found: 2011-04,Baidu = 201227000
+Data found: 2011-05,Baidu = 211060000
+Data found: 2011-06,Baidu = 215086000
+Data found: 2011-07,Baidu = 225898000
+Data found: 2010-05,eBay = 252803000
+Data found: 2010-06,eBay = 217671000
+Data found: 2010-07,eBay = 217586000
+Data found: 2010-08,eBay = 222435000
+Data found: 2010-09,eBay = 221574000
+Data found: 2010-10,eBay = 222283000
+Data found: 2010-11,eBay = 229915000
+Data found: 2010-12,eBay = 225280000
+Data found: 2011-01,eBay = 226079000
+Data found: 2011-02,eBay = 211945000
+Data found: 2011-03,eBay = 218809000
+Data found: 2011-04,eBay = 215085000
+Data found: 2011-05,eBay = 225893000
+Data found: 2011-06,eBay = 223520000
+Data found: 2011-07,eBay = 224511000
+Data found: 2010-05,CBS = 211496000
+Data found: 2010-06,CBS = 204162000
+Data found: 2010-07,CBS = 209288000
+Data found: 2010-08,CBS = 214875000
+Data found: 2010-09,CBS = 223388000
+Data found: 2010-10,CBS = 222329000
+Data found: 2010-11,CBS = 229265000
+Data found: 2010-12,CBS = 221362000
+Data found: 2011-01,CBS = 226135000
+Data found: 2011-02,CBS = 210949000
+Data found: 2011-03,CBS = 234656000
+Data found: 2011-04,CBS = 217686000
+Data found: 2011-05,CBS = 220230000
+Data found: 2011-06,CBS = 214019000
+Data found: 2011-07,CBS = 217962000
+Data found: 2010-05,Ask Network = 215039000
+Data found: 2010-06,Ask Network = 208313000
+Data found: 2010-07,Ask Network = 211622000
+Data found: 2010-08,Ask Network = 216371000
+Data found: 2010-09,Ask Network = 221821000
+Data found: 2010-10,Ask Network = 225397000
+Data found: 2010-11,Ask Network = 233155000
+Data found: 2010-12,Ask Network = 233444000
+Data found: 2011-01,Ask Network = 234884000
+Data found: 2011-02,Ask Network = 220019000
+Data found: 2011-03,Ask Network = 236956000
+Data found: 2011-04,Ask Network = 220341000
+Data found: 2011-05,Ask Network = 224060000
+Data found: 2011-06,Ask Network = 212548000
+Data found: 2011-07,Ask Network = 217771000
+Data found: 2010-05,Glam Media = 164189000
+Data found: 2010-06,Glam Media = 167420000
+Data found: 2010-07,Glam Media = 173248000
+Data found: 2010-08,Glam Media = 186414000
+Data found: 2010-09,Glam Media = 179892000
+Data found: 2010-10,Glam Media = 181888000
+Data found: 2010-11,Glam Media = 197937000
+Data found: 2010-12,Glam Media = 196886000
+Data found: 2011-01,Glam Media = 202299000
+Data found: 2011-02,Glam Media = 194173000
+Data found: 2011-03,Glam Media = 204134000
+Data found: 2011-04,Glam Media = 200186000
+Data found: 2011-05,Glam Media = 213528000
+Data found: 2011-06,Glam Media = 203630000
+Data found: 2011-07,Glam Media = 200757000
+
+Entities ignored:
+114LA.COM, 120ASK.COM, 123GREETINGS.COM, 1616.NET, 21CN Corporation Limited, 2345.COM, 24H.COM.VN, 265.COM, 2CH.NET, 2DPLAY.COM, 2SHARED.COM, 39.NET, 4SHARED.COM, 51.COM, 51JOB.COM, 51MOLE.COM, 52PK.COM, 56.COM, 58.COM, 6.CN, 6ARAB.COM, 6RB.COM, 6RBTOP.COM, 7K7K.COM, 8684.CN, 8D.CN, ABCNEWS DIGITAL, ABOUTUS.ORG, AL-WLID.COM, ALTOOLS.COM, ARABSEYES.COM, ARCHIVE.ORG, ARD Sites, ARTISTdirect Network, AT&T Inc., AT&T Interactive Network, AVAST.COM, AVG Technologies, AVS4YOU.COM, Accor, AccuWeather Sites, Acer Inc., Activision Blizzard, Adobe Sites, Adoos Classifieds S.L., Advance Internet, Agricultural Bank of China, Air France-KLM Group, Alibaba.com Corporation, Aljazeera, Allesklar.com AG Sites, AlloCine Sites, Alloy Digital Network, Altice, Amadeus Global Travel Distribution, American Airlines, American Express, AmericanGreetings Property, Ancestry.com Inc., Answers.com Sites, Arcandor AG, Art.com Sites, ArticlesBase.com Sites, Asahi Shimbun, Atrinsic Inc., Avon Products Inc., Axel Springer AG, B5Media, BAAMBOO.COM, BABYLON.COM, BADONGO.COM, BADOO.COM, BAIXING.COM, BBC Sites, BBVOD.NET, BEEMP3.COM, BIGMIR.NET, BITCOMET.COM, BLOGBUS.COM, BLOGGUM.COM, BMW, BNP Paribas, BOOSJ.COM, BRG8.COM, BTJUNKIE.ORG, BUECHER.DE, BUZZMEDIA, BabyCenter Network, Bank of America, Barclays Bank, Barnes & Noble, Batanga, Bauer Consumer Media, BearShare (App), Belgacom Group, Belo, Benesse Corporation, Berkshire Hathaway Inc., Best Buy Sites, Bestofmedia Group, Betawave Media, Big Fish Games Sites, Big Lead Sports By FSV, Bigpoint, BitComet (App), BitTorrent Network, Blockbuster Inc., Bokee Sites, Bravenet Web Services, Break Media, British Telecommunications, BuscaPe.com Inc., CA.GOV, CANALBLOG.COM*, CBC-Radio Canada Sites, CCLEANER.COM, CCM-Benchmark, CDC Corporation Sites, CDUNIVERSE.COM, CENTERBLOG.NET, CHINA.COM.CN, CITY-DATA.COM, CJ Group, CLASSESINTERNATIONAL.COM, CONDUIT.COM, COOKPAD.COM, COOKS.COM, COOLCHASER.COM, COOLS4U.COM, COOLSAVINGS.COM, CQVIP.COM, CRI Online Sites, CRSKY.COM, CSDN.NET Sites, CT10000.COM, CYBERLINK.COM, Cablevision, CafeMom Network, CafePress, Camfrog (App), Canoe Network, Canon Inc., Capital One, CareerBuilder LLC, Carrefour Group, Channel4, China Central Television Sites, China Construction Bank, China Economic Net, China Mobile Sites, China Telecom Sites, Chinabyte Technology Co., Chosun Ilbo, Cisco Systems Inc., Citigroup, Citrix Systems Inc., CityGrid Media, Classified Ventures, Clear Channel Online, Clearspring Technologies, Coca-Cola Company, CollegeHumor Media, Comcast Corporation, Compare Infobase Limited, Conde Nast Digital, Connexus, Corel Corporation, Costco Wholesale Corporation, Coupons Inc., Cox Enterprises Inc., CraveOnline (Evolve Media Corp.), Credit Agricole, Ctrip.com International, CyberAgent, Cyworld, DAILYMOTION.COM, DANGDANG.COM, DEEZER.COM, DEPOSITFILES.COM, DEVIANTART.COM, DIPAN.COM, DIRECT.GOV.UK, DIVSHARE.COM, DLTK Network Sites, DMGT (exc Mail Online), DOMAINTOOLS.COM, DOUBAN.COM, DOWNLOAD3000.COM, DUOTE.COM, DUOTEGAME.COM, DUOWAN.COM, DeNA Co. Ltd., Dell, Delta Airlines, Demand Media, Department of Commerce, Deutsche Bahn, Deutsche Post AG Sites, Deutsche Telekom, Directv Sites, Discover Financial Services, Discovery Digital Media Sites, Disney Online, Dogan Online, Dominion Enterprises, Domino's Pizza Inc., Dong-A Ilbo, Dow Jones & Company, Dwango Co. Ltd., E.W. Scripps, EA Online, EASY-SHARE.COM, ED.GOV, ELAANA.COM, EMP3WORLD.COM, ESNIPS.COM, ESPN, EUROPA.EU, EVA.VN, EVERSAVE.COM, EZINEARTICLES.COM, Earthlink, Eastman Kodak, EasyGroup, Ekspress Grupp, Encyclopaedia Britannica, Eniro Sites, Estee Lauder Sites, Eutelia, Everyday Health, Excite Japan Sites, Expedia Inc, Experian Interactive, FANBOX.COM, FC2 inc., FILEHIPPO.COM, FILESTUBE.COM, FLASHGAMES247.COM, FRIENDSTER.COM, FUNSHION.COM, Fairfax Media, FedEx, Federated Media Publishing, First Choice Holidays PLC, Forbes Digital, Ford Motor Company, FortuneCity Global Community, ForumCommunity, Fotosearch LLC, Fotovista Group, Fox Interactive Media, France Televisions Interactive, Freechal INC., FrostWire, Fuji Television Network, GAME2.CN, GAMEBTO.COM, GAMES2GIRLS.COM, GAMETOP.COM, GAMEZER.COM, GANJI.COM, GARENA.COM, GGL Media, GMO Internet Group, GN Multicultural (Evolve Media Corp.), GRAAAM.COM, GSN Games, GYYX.CN, GameDuell, GameStop, Gameforge Sites, Gannett Sites, Ganz, Gap Inc., Gawker Media, Geeknet, General Mills, General Motors, Getty Images Inc, GiantRealm, Globe7, Globo, GoNamic GmbH, Gorilla Nation (Evolve Media Corp), Gretech Corp., Groupe Amaury, Groupe Auchan, Groupe Casino, Groupe Figaro, Groupe LDLC.com, Groupe Lagardere, Groupe M6, Groupe PPR, Groupe Pages Jaunes, Groupe SNCF, Groupe Spir Communication, Groupe TF1, Gruner+Jahr Sites, Grupo Abril, Grupo Carso, Grupo Clarin, Grupo Intercom, Grupo Planeta Sites, Grupo Santander, Gruppo Editoriale Espresso, Gruppo Fiat, Gruppo Mediaset, Guardian Media Group, Gurunavi, H&R Block, HAWAAWORLD.COM, HC International Inc., HENHAOJI.COM, HEXUN.COM, HI5.COM, HMV, HONGXIU.COM, HSBC, Habbo Network, Hallmark, Hankook Ilbo, Hankyoreh Newspaper, Hatena, HealthCentral, HealthGrades, Healthline, Hearst Corporation, Hearst Digital Media, Hewlett Packard, Hi-Media Group, Hilton Hotels, Hispavista Sites, Hollywood.com Sites, Home Retail Group, HomeAway, Honda, HotChalk, Hubert Burda Media, Hulu, Hutchison Whampoa, Hyundai Motors Inc., Hyves, IBM, IDG Network, IDT Corporation, IKEA, IKM Internet Kaufmarkt, IMAGEVENUE.COM, INFO.COM*, ING Group, IP138.COM, IRS.GOV, IT&IS Siglo XXI, ITV Sites, Iliad - Free.fr Sites, Imaginova Network, Impress Group, IncrediMail Ltd, Indeed, Independent News & Media, Indian Railways, Infospace Network, InnoGames GmbH, Intel, Intelius, InterContinental Hotels Group, Interactive One, Interdots International, Internet Brands Inc., Internet Initiative Japan, Interpark, Intuit, JCPenney Sites, JEDDAHBIKERS.COM, JEERAN.COM, JMnet (JoongAng Media Network), JPMorgan Chase Property, JSOFTJ.COM, Jabbar Internet Group, Japan Post Group, Job.com Sites, KAIXIN001.COM, KAPOOK.COM, KBB.COM, KBS (Korean Broadcasting System), KDDI Corporation, KEENFINDER.COM, KIEV.UA, KINO.TO, KONGREGATE.COM, KOOORA.COM, KOOWO.COM, KPN, KT Group, KUWAITUP.COM, KaiserGames GmbH, Kakaku.com Inc., Kayak.com Network, Kiabi, Kingfisher, Kingsoft Corp, Kipubli, Kohls Corporation, Korea Economic Daily, Kraft Foods, KuGou Inc., Kuaiche.com Sites, L'Oreal Sites, LEGACY.COM*, LEO.ORG, LETITBIT.NET, LG Group, LIMEWIRE.COM, LINKEDIN.COM, LIVEINTERNET.RU, LIVESCORE.COM, LOCALBILLING.COM, LOWES.COM, LVMH Sites, La Poste, Le Monde Sites, LeGuide.com Sites, Lee Enterprises Incorporated, Liberty Media Holding Corporation, Limitedbrands, Live Nation Network, Lloyds Banking Group plc, Lojas Americanas, Lotte Group, Lycos Global Search And Media, M5ZN.COM, MADE-IN-CHINA.COM, MANTA.COM, MAPBAR.COM, MBC (Munhwa Broadcasting Corp.), MEDIAFIRE.COM, MEGAUPLOAD.COM, MEGAVIDEO.COM, MERCHANTCIRCLE.COM, METEOFRANCE.COM, METRO Group, MEXAT.COM, MIH Limited, MINICLIP.COM, MINIJUEGOS.COM, MININOVA.ORG, MIXI Inc., MONOGRAFIAS.COM, MOZOOK.COM, MSGPLUSLIVE.NET, MTHAI.COM, MULTIPLY.COM, MUSICA.COM, MYHOTCOMMENTS.COM, Macy's Inc., Maeil Business Newspaper, Mainichi Shimbun, Mapsolute, Marriott, Match.com Sites, Mattel Sites, McAfee.com Sites, McClatchy Corporation, McDonald's, Media General Sites, MediaNews Group, Meebo Messenger IM, MeetIC - Match, MercadoLibre, Meredith Women's Network, Metacafe, Miarroba Networks, Michelin Group, Midasplayer.com Ltd., Mio.tv Network, Moneysupermarket.com Financial Group, Monster Inc., Move Inc. Real Estate Sites, MyLife.com Sites, NAMIPAN.COM, NASA.GOV, NASZA-KLASA.PL, NBC Universal, NEC Corporation, NERO.COM, NETFLIX.COM, NFL Internet Group, NHACCUATUI.COM, NHK Group, NHL Network, NHN Corporation, NIC.IN, NIH, NIKKEI, NING.COM, NOWCOM, NTT Group, National Geographic Sites, National Lampoon Humor Network, Naukri, Nestle, NetEase.com Inc., NetShelter Technology Media, Network 18, New York Times Digital, NewIPs, News Interactive Pty Limited, News International, News World Communications Inc., Nexon Corporation, NextRadio TV, Nextag.com Sites, Nifty Corporation, Nike, Nintendo Co., Nippon Television Network, Nissan, Nokia, Notrefamille.com Sites, OHMYNEWS.COM, OLX Inc., OPENDNS.COM, OPENV.COM*, ORBITDOWNLOADER.COM, OSCAR.COM, OVH Sites, OWNSKIN.COM, OYUNLAR1.COM, OZQ8.COM, Oak Pacific Interactive Sites, Oberon Media Inc, Office Depot, Onlinedown Sites, Opera Software, Oracle Corporation, Orange Sites, Orbitz Worldwide, Oricon, Otto Gruppe, Oversee.net Network, PANDORA.COM, PANDORA.TV, PANET.CO.IL, PBASE.COM, PBS, PCPOP Sites, PChome Online, PConline.cn Sites, PHOTOFUNIA.COM, PHPNUKE.ORG, PICNIK.COM, PIPI.CN, PLAYLIST.COM, PLENTYOFFISH.COM, POMOHO.COM, PPLive Inc., PPStream Inc., PSA Peugeot Citro\xEBn, PSICOFXP.COM, PYZAM.COM, PalTalk, Panasonic Group, Pangea Media, PartyGaming, Pearson Education, People's Daily online, Pepsi Co., Photobucket.com LLC, PlasmaNet Sites, Play.com Sites, PokerStars, Populis, Portugal Telecom, Postmedia Network, PriceGrabber, Priceline.com Incorporated, Prisa, ProSiebenSat1 Sites, Procter & Gamble, Proto Corporation, Publieke Omroep, QIANQIAN.COM, QIDIAN.COM, QQ163.COM, QVOD.COM, Qihoo.com Sites, RAPIDLIBRARY.COM, RARLAB.COM, RAYFILE.COM, RAYLI.COM.CN, RBC Financial Group, RCS Media Group, READNOVEL.COM, REGNOW.COM, RH Donnelley, ROGERS, RORO44.COM, RTL Group Sites, Rakuten Inc, Rambler Media, RapidShare AG, Reader's Digest Digital Network, Real.com Network, Recruit Co. Ltd., Rediff.com India Ltd, Reed Business Information, Reed Elsevier Group, Research In Motion Limited, Reuters Sites, Rising Corp. Limited, RosBusinessConsulting, Roularta Media Group, Rue du Commerce, Ryanair, SBS (Seoul Broadcasting System), SCRIBD.COM, SEDO Sites, SENDSPACE.COM, SH.CN, SHERV.NET, SHOPLOCAL.COM, SHOUTMIX.COM, SHVOONG.COM, SINA Corporation, SK Group, SLIDESHARE.NET, SNDA Sites, SOFTBANK Group, SOFTPEDIA.COM, SONGS.PK, SPB.RU, SPLINDER.COM, SPRINGERLINK.COM, STARTIMES2.COM, STARTVG.COM, SUMOTORRENT.COM, SWEETIM.COM, Sakura Internet, Samsung Group, Samurai Factory Inc., Sankei Shimbun, Sanoma Group, Schibsted (Anuntis-Infojobs-20minutos), Scripps Networks Interactive Inc., SeaRainbow Holding Corp, Sears Sites, Seat Pagine Gialle, Seesaa, Seiko Epson Corp, Seoul Newspaper, Sheknows (Evolve Media Corp.), Shopzilla.com Sites, Sky Sites, SkyCN, Skyrock, Slide Sites, Societe Generale, Softissimo, Sohu.com Inc., Sonic Solutions, Sonico Inc., Sony Online, SouFun.com Limited, Source Interlink Companies, Southwest Airlines Co., Sparkassen-Finanzgruppe, Spiegel Verlag, Spil Games, Sprint Nextel Corporation, Stage6 by DivX, Staples.com Sites, Stateless Systems, Steam (App), Sugar Inc - PopSugar Network, Sun Microsystems Inc., Superpages.com Network, Symantec, Systemax Inc., TAGSTORY.COM, TALEO.NET, TARINGA.NET, TBS Group, TD Bank Financial Group, TE3P.COM, TGBUS.COM, THEPIRATEBAY.ORG, TIEXUE.NET, TIMEANDDATE.COM, TIMNHANH.COM, TOGGLE.COM, TOM.COM Limited, TOPSHAREWARE.COM, TORRENTPORTAL.COM, TORRENTREACTOR.NET, TORRENTS.RU, TORRENTZ.COM, TPI Paginas Amarillas, TQ Digital Entertainment, TUI Group, TV Guide Online Network, TWITTER.COM, Take 2 Interactive, Target Corporation, TaxACT, Tchibo, Technorati Media, Tele2 Sites, Telecom Italia, Telegraaf Media Groep, Telegraph Media Group, Televisa Interactive Media, Telstra Corporation Limited, Terra - Telefonica, Tesco Stores, The Home Depot Inc., The Mozilla Organization, The Royal Bank Of Scotland, The Washington Post Company, The Weather Network Meteomedia, Thomson Reuters, Ticketmaster, Time Warner (Excl. Turner/WB), Times Internet Limited, Tiscali Sites, Toshiba Corp, Toyota, Toysrus Sites, Trader Media Group, Transcontinental Digital, Travelocity, Travian Games, Trend Micro Inc., Tribune Interactive, Trinity Mirror Group, Trovit Sites, Tudou Sites, Turner Digital, Twenga, UGO Entertainment, UOL, UPLOADED.TO, UPS Sites, UPTODOWN.COM, URBANDICTIONARY.COM, USAFIS.ORG, USEN Group, USPS.COM, UUSEE.COM, UUU9.COM, Ubisoft Entertainment, Unilever, Unister GmbH, United Business Media, United Online Inc, United-Internet Sites, Universia, VATGIA.COM, VEOH.COM, VUI.VN, ValueClick Sites, Valve Corporation, Vector Inc., Veoh (App), Verizon Communications Corporation, Verlagsgruppe Georg von Holtzbrinck, VeryCD, Viacom Digital, Virgin Group, Virgin Media, VistaPrint Sites, Viva street Classifieds, Vivendi, Vkontakte, Vocento, Vodafone Group, Volkswagen, Vuze Sites, WANMEI.COM, WARESEEKER.COM, WEBHATTI.COM, WIKIHOW.COM, WIKIMAPIA.ORG, WILEY.COM, WIND Telecomunicazioni, WORDREFERENCE.COM, WOYO.COM, Wal-Mart, Walgreen Co., Weather Channel The, WeatherBug Property, WebMD Health, WebMediaBrands Inc., Weblogs SL Sites, Weight Watchers International, Wells Fargo, WetterOnline GmbH, WhitePages, Wikia Sites, Wikio Group, WildTangent Media, WordPress, World of Warcraft (App), WorldNow Sites, XANGA.COM, XAT.COM, XE.COM, XICI.NET, Xacti Corporation, Xinhua News Agency, Xunlei Networking, Y8.COM, YAM.COM, YAMAHA, YE-SOFT.NET, YELP.COM, YOKA.COM, YOURFILEHOST.COM, Yahoo, Yandex Sites, Yellow Pages Group, YellowBook Network, Yomiuri Group, Youku Inc., Youthology Kids (Evolve Media Corp.), Yum! Brands Inc., ZHAOPIN.COM, ZHCW.COM, ZSHARE.NET, ZVue Networks, Zazzle, Ziff Davis, craigslist inc., eBuddy Sites, eMule (App), eNet, edaily, freenet AG, iEUROP Sites, iG Sites, iVillage.com: The Womens Network, imeem, mIRC, spreadshirt
+
+
+Updates found, rewrite master file 'history_comscore_UV_properties.csv'
+
+WriteDataAnalytics
+
+2008-01,-,AS,-,-,19.4,59964000
+2008-01,-,C,-,-,0.6,581000
+2008-01,-,EU,-,-,39.1,91036000
+2008-01,-,I,-,-,19.2,5138000
+2008-01,-,LA,-,-,31.1,18341000
+2008-01,-,MA,-,-,28.9,11549000
+2008-01,-,NA,-,-,33.5,61664000
+2008-01,-,W,-,-,29.4,242554000
+2008-02,-,AS,-,-,19.5,59948000
+2008-02,-,C,-,-,0.7,695000
+2008-02,-,EU,-,-,37.9,87975000
+2008-02,-,I,-,-,18.8,5065000
+2008-02,-,LA,-,-,34.2,20286000
+2008-02,-,MA,-,-,28.3,11327000
+2008-02,-,NA,-,-,33.2,61218000
+2008-02,-,W,-,-,29.3,240754000
+2008-03,-,AS,-,-,19.8,62380000
+2008-03,-,C,-,-,0.7,669000
+2008-03,-,EU,-,-,39.1,92283000
+2008-03,-,I,-,-,18.7,5200000
+2008-03,-,LA,-,-,40.4,24542000
+2008-03,-,MA,-,-,32.2,13337000
+2008-03,-,NA,-,-,34.0,63518000
+2008-03,-,W,-,-,30.5,256061000
+2008-04,-,AS,-,-,20.1,63963000
+2008-04,-,C,-,-,2.2,2288000
+2008-04,-,EU,-,-,39.3,93642000
+2008-04,-,I,-,-,17.9,5079000
+2008-04,-,LA,-,-,42.8,26347000
+2008-04,-,MA,-,-,32.3,13634000
+2008-04,-,NA,-,-,33.8,63827000
+2008-04,-,W,-,-,30.8,261414000
+2008-05,-,AS,-,-,19.7,63560000
+2008-05,-,C,-,-,0.9,994000
+2008-05,-,EU,-,-,39.7,95234000
+2008-05,-,I,-,-,18.2,5264000
+2008-05,-,LA,-,-,43.2,27002000
+2008-05,-,MA,-,-,32.9,14139000
+2008-05,-,NA,-,-,34.1,63186000
+2008-05,-,W,-,-,30.8,263120000
+2008-06,-,AS,-,-,20.1,65764000
+2008-06,-,C,-,-,0.9,958000
+2008-06,-,EU,-,-,37.0,89427000
+2008-06,-,I,-,-,20.4,5979000
+2008-06,-,LA,-,-,41.2,26099000
+2008-06,-,MA,-,-,27.3,11941000
+2008-06,-,NA,-,-,31.5,58272000
+2008-06,-,W,-,-,29.2,251502000
+2008-07,-,AS,-,-,17.2,65838000
+2008-07,-,C,-,-,0.1,232000
+2008-07,-,EU,-,-,32.4,86905000
+2008-07,-,I,-,-,19.2,5720000
+2008-07,-,LA,-,-,33.6,23220000
+2008-07,-,MA,-,-,24.7,11016000
+2008-07,-,NA,-,-,31.1,57347000
+2008-07,-,W,-,-,25.7,244326000
+2008-08,-,AS,-,-,17.2,66895000
+2008-08,-,C,-,-,0.8,1382000
+2008-08,-,EU,-,-,31.0,83931000
+2008-08,-,I,-,-,20.5,6217000
+2008-08,-,LA,-,-,38.2,26847000
+2008-08,-,MA,-,-,25.1,11371000
+2008-08,-,NA,-,-,32.4,59494000
+2008-08,-,W,-,-,25.9,248539000
+2008-09,-,AS,-,-,17.2,68291000
+2008-09,-,C,-,-,1.6,2735000
+2008-09,-,EU,-,-,35.9,98304000
+2008-09,-,I,-,-,19.6,6025000
+2008-09,-,LA,-,-,40.4,28886000
+2008-09,-,MA,-,-,27.2,12553000
+2008-09,-,NA,-,-,34.7,64075000
+2008-09,-,W,-,-,28.0,272109000
+2008-10,-,AS,-,-,16.3,65703000
+2008-10,-,C,-,-,1.2,2037000
+2008-10,-,EU,-,-,37.2,102955000
+2008-10,-,I,-,-,18.6,5799000
+2008-10,-,LA,-,-,38.6,27974000
+2008-10,-,MA,-,-,29.3,13790000
+2008-10,-,NA,-,-,36.0,66785000
+2008-10,-,W,-,-,28.2,277208000
+2008-11,-,AS,-,-,16.7,68371000
+2008-11,-,C,-,-,1.2,2185000
+2008-11,-,EU,-,-,38.0,106199000
+2008-11,-,I,-,-,19.9,6286000
+2008-11,-,LA,-,-,37.2,27416000
+2008-11,-,MA,-,-,28.9,13850000
+2008-11,-,NA,-,-,35.1,65133000
+2008-11,-,W,-,-,28.2,280969000
+2008-12,-,AS,-,-,15.9,66026000
+2008-12,-,C,-,-,1.3,2286000
+2008-12,-,EU,-,-,37.3,105318000
+2008-12,-,I,-,-,20.0,6421000
+2008-12,-,LA,-,-,30.4,22769000
+2008-12,-,MA,-,-,31.0,15103000
+2008-12,-,NA,-,-,34.5,63782000
+2008-12,-,W,-,-,27.1,272998000
+2009-01,-,AS,-,-,16.8,71203000
+2009-01,-,C,-,-,1.0,1825000
+2009-01,-,EU,-,-,39.4,112449000
+2009-01,-,I,-,-,20.8,6764000
+2009-01,-,LA,-,-,29.6,22558000
+2009-01,-,MA,-,-,31.5,15630000
+2009-01,-,NA,-,-,36.6,67971000
+2009-01,-,W,-,-,28.4,289811000
+2009-02,-,AS,-,-,16.4,70862000
+2009-02,-,C,-,-,1.1,2002000
+2009-02,-,EU,-,-,38.5,117828000
+2009-02,-,I,-,-,20.0,6587000
+2009-02,-,LA,-,-,33.9,27440000
+2009-02,-,MA,-,-,24.8,18765000
+2009-02,-,NA,-,-,35.5,65855000
+2009-02,-,W,-,-,27.9,300751000
+2009-03,-,AS,-,-,17.8,78009000
+2009-03,-,C,-,-,1.4,2752000
+2009-03,-,EU,-,-,40.9,126474000
+2009-03,-,I,-,-,21.0,7014000
+2009-03,-,LA,-,-,41.5,34229000
+2009-03,-,MA,-,-,28.2,21558000
+2009-03,-,NA,-,-,36.0,66878000
+2009-03,-,W,-,-,29.9,327148000
+2009-04,-,AS,-,-,17.0,76024000
+2009-04,-,C,-,-,1.6,3115000
+2009-04,-,EU,-,-,38.3,119578000
+2009-04,-,I,-,-,20.2,6807000
+2009-04,-,LA,-,-,39.8,33400000
+2009-04,-,MA,-,-,26.0,20133000
+2009-04,-,NA,-,-,37.8,70908000
+2009-04,-,W,-,-,28.9,320043000
+2009-05,-,AS,-,-,16.6,75595000
+2009-05,-,C,-,-,1.5,2936000
+2009-05,-,EU,-,-,37.6,118874000
+2009-05,-,I,-,-,21.0,7168000
+2009-05,-,LA,-,-,39.6,33870000
+2009-05,-,MA,-,-,24.9,19529000
+2009-05,-,NA,-,-,36.8,69387000
+2009-05,-,W,-,-,28.3,317255000
+2009-06,-,AS,-,-,16.5,76239000
+2009-06,-,C,-,-,2.2,4566000
+2009-06,-,EU,-,-,35.2,112270000
+2009-06,-,I,-,-,21.7,7522000
+2009-06,-,LA,-,-,38.3,33263000
+2009-06,-,MA,-,-,20.8,16489000
+2009-06,-,NA,-,-,34.4,64678000
+2009-06,-,W,-,-,26.7,302940000
+2009-07,-,AS,-,-,15.9,74990000
+2009-07,-,C,-,-,1.8,3839000
+2009-07,-,EU,-,-,33.7,108552000
+2009-07,-,I,-,-,22.6,7908000
+2009-07,-,LA,-,-,33.1,29298000
+2009-07,-,MA,-,-,20.6,16502000
+2009-07,-,NA,-,-,34.9,66142000
+2009-07,-,W,-,-,25.7,295484000
+2009-08,-,AS,-,-,16.2,77365000
+2009-08,-,C,-,-,1.6,3547000
+2009-08,-,EU,-,-,34.2,111190000
+2009-08,-,I,-,-,23.7,8398000
+2009-08,-,LA,-,-,36.8,33186000
+2009-08,-,MA,-,-,21.9,17816000
+2009-08,-,NA,-,-,35.6,68084000
+2009-08,-,W,-,-,26.4,307641000
+2009-09,-,AS,-,-,16.1,77948000
+2009-09,-,C,-,-,1.3,2882000
+2009-09,-,EU,-,-,37.4,123062000
+2009-09,-,I,-,-,23.7,8480000
+2009-09,-,LA,-,-,38.4,35264000
+2009-09,-,MA,-,-,22.0,18063000
+2009-09,-,NA,-,-,37.3,71661000
+2009-09,-,W,-,-,27.6,325998000
+2009-10,-,AS,-,-,17.0,83671000
+2009-10,-,C,-,-,1.6,3702000
+2009-10,-,EU,-,-,39.0,128647000
+2009-10,-,I,-,-,25.7,9287000
+2009-10,-,LA,-,-,39.0,36440000
+2009-10,-,MA,-,-,26.9,22353000
+2009-10,-,NA,-,-,38.2,73452000
+2009-10,-,W,-,-,29.0,344563000
+2009-11,-,AS,-,-,16.5,82379000
+2009-11,-,C,-,-,1.6,3718000
+2009-11,-,EU,-,-,39.0,129798000
+2009-11,-,I,-,-,25.8,9437000
+2009-11,-,LA,-,-,38.4,36495000
+2009-11,-,MA,-,-,26.2,22083000
+2009-11,-,NA,-,-,38.5,75050000
+2009-11,-,W,-,-,28.7,345805000
+2009-12,-,AS,-,-,16.7,84201000
+2009-12,-,C,-,-,1.5,3657000
+2009-12,-,EU,-,-,38.6,129763000
+2009-12,-,I,-,-,26.3,9696000
+2009-12,-,LA,-,-,32.5,31391000
+2009-12,-,MA,-,-,28.3,24058000
+2009-12,-,NA,-,-,39.1,77607000
+2009-12,-,W,-,-,28.4,347020000
+2010-01,-,AS,-,-,17.4,89049000
+2010-01,-,C,-,-,1.4,3351000
+2010-01,-,EU,-,-,40.3,136935000
+2010-01,-,I,-,-,27.4,10216000
+2010-01,-,LA,-,-,33.5,32834000
+2010-01,-,MA,-,-,27.2,23411000
+2010-01,-,NA,-,-,41.1,82489000
+2010-01,-,W,-,-,29.5,364719000
+2010-02,-,AS,-,-,15.3,72378000
+2010-02,-,C,-,-,1.2,2935000
+2010-02,-,EU,-,-,39.2,133807000
+2010-02,-,I,-,-,26.4,9865000
+2010-02,-,LA,-,-,36.7,36414000
+2010-02,-,MA,-,-,24.9,21680000
+2010-02,-,NA,-,-,39.9,80940000
+2010-02,-,W,-,-,28.7,345218000
+2010-03,-,AS,-,-,16.2,77678000
+2010-03,-,C,-,-,1.4,3443000
+2010-03,-,EU,-,-,40.8,140834000
+2010-03,-,I,-,-,27.7,10516000
+2010-03,-,LA,-,-,43.1,43574000
+2010-03,-,MA,-,-,28.3,24962000
+2010-03,-,NA,-,-,41.0,83696000
+2010-03,-,W,-,-,30.4,370744000
+2010-04,-,AS,-,-,15.9,77535000
+2010-04,-,C,-,-,1.6,4081000
+2010-04,-,EU,-,-,41.4,144087000
+2010-04,-,I,-,-,28.4,10923000
+2010-04,-,LA,-,-,43.2,44286000
+2010-04,-,MA,-,-,28.1,25146000
+2010-04,-,NA,-,-,40.9,83792000
+2010-04,-,W,-,-,30.4,374846000
+2010-05,-,AS,-,-,16.3,80502000
+2010-05,-,C,-,-,1.0,2700000
+2010-05,-,EU,-,-,43.0,150313000
+2010-05,-,I,-,-,30.1,11675000
+2010-05,-,LA,-,-,44.1,45790000
+2010-05,-,MA,-,-,29.2,26399000
+2010-05,-,NA,-,-,41.5,85929000
+2010-05,-,W,-,-,31.3,388932000
+2010-06,-,AS,-,-,16.1,80269000
+2010-06,-,C,-,-,1,2613000
+2010-06,-,EU,-,-,43,151013000
+2010-06,-,I,-,-,31.6,12373000
+2010-06,-,LA,-,-,41.8,43826000
+2010-06,-,MA,-,-,25.6,23611000
+2010-06,-,NA,-,-,38.9,80393000
+2010-06,-,W,-,-,30.2,379112000
+2010-07,-,AS,-,-,15.9,80260000
+2010-07,-,C,-,-,0.9,2536000
+2010-07,-,EU,-,-,39.3,138975000
+2010-07,-,I,-,-,32.4,12828000
+2010-07,-,LA,-,-,35.9,38111000
+2010-07,-,MA,-,-,24.4,22991000
+2010-07,-,NA,-,-,39,79888000
+2010-07,-,W,-,-,28.5,360225000
+2010-08,-,AS,-,-,16.2,83015000
+2010-08,-,C,-,-,1,2829000
+2010-08,-,EU,-,-,39.6,140560000
+2010-08,-,I,-,-,33,13187000
+2010-08,-,LA,-,-,39.9,42303000
+2010-08,-,MA,-,-,24.4,26220000
+2010-08,-,NA,-,-,39.7,81295000
+2010-08,-,W,-,-,29,373392000
+2010-09,-,AS,-,-,16.1,83119000
+2010-09,-,C,-,-,1.1,3181000
+2010-09,-,EU,-,-,43.5,155438000
+2010-09,-,I,-,-,33.3,13433000
+2010-09,-,LA,-,-,42.2,45265000
+2010-09,-,MA,-,-,26.5,28749000
+2010-09,-,NA,-,-,41.9,85606000
+2010-09,-,W,-,-,30.8,398178000
+2010-10,-,AS,-,-,16,83406000
+2010-10,-,C,-,-,1.1,3156000
+2010-10,-,EU,-,-,44.4,159384000
+2010-10,-,I,-,-,33.5,13674000
+2010-10,-,LA,-,-,41.6,45152000
+2010-10,-,MA,-,-,30.1,33049000
+2010-10,-,NA,-,-,42.7,87359000
+2010-10,-,W,-,-,31.3,408350000
+2010-11,-,AS,-,-,15.7,82736000
+2010-11,-,C,-,-,1.1,3108000
+2010-11,-,EU,-,-,45.8,165020000
+2010-11,-,I,-,-,33.1,13614000
+2010-11,-,LA,-,-,42.4,46617000
+2010-11,-,MA,-,-,27.6,30711000
+2010-11,-,NA,-,-,42,85732000
+2010-11,-,W,-,-,31.3,410816000
+2010-12,-,AS,-,-,15.2,80958000
+2010-12,-,C,-,-,1.2,3372000
+2010-12,-,EU,-,-,43.6,157837000
+2010-12,-,I,-,-,33.7,14004000
+2010-12,-,LA,-,-,34.8,38738000
+2010-12,-,MA,-,-,27.4,30852000
+2010-12,-,NA,-,-,42.7,87087000
+2010-12,-,W,-,-,29.9,395472000
+2011-01,-,AS,-,-,15.9,85593000
+2011-01,-,C,-,-,1,3048000
+2011-01,-,EU,-,-,46.4,168571000
+2011-01,-,I,-,-,35.9,15067000
+2011-01,-,LA,-,-,35,39446000
+2011-01,-,MA,-,-,28.2,32113000
+2011-01,-,NA,-,-,43.2,88233000
+2011-01,-,W,-,-,31,413957000
+2011-02,-,AS,-,-,14.2,77468000
+2011-02,-,C,-,-,0.9,2664000
+2011-02,-,EU,-,-,41.6,150775000
+2011-02,-,I,-,-,30.6,12960000
+2011-02,-,LA,-,-,36.3,41411000
+2011-02,-,MA,-,-,25.8,29722000
+2011-02,-,NA,-,-,39.4,80038000
+2011-02,-,W,-,-,28.3,379415000
+2011-03,-,AS,-,-,14.7,80997000
+2011-03,-,C,-,-,0.9,2854000
+2011-03,-,EU,-,-,43.1,156913000
+2011-03,-,I,-,-,30.4,13011000
+2011-03,-,LA,-,-,41.3,47583000
+2011-03,-,MA,-,-,28,32599000
+2011-03,-,NA,-,-,40.1,81918000
+2011-03,-,W,-,-,29.6,400011000
+2011-04,-,AS,-,-,14,77891000
+2011-04,-,C,-,-,0.9,2866000
+2011-04,-,EU,-,-,40.8,149211000
+2011-04,-,I,-,-,29.6,12801000
+2011-04,-,LA,-,-,38.7,45108000
+2011-04,-,MA,-,-,25.7,30272000
+2011-04,-,NA,-,-,38,78234000
+2011-04,-,W,-,-,27.9,380716000
+2011-05,-,AS,-,-,15.1,84666000
+2011-05,-,C,-,-,1.1,3492000
+2011-05,-,EU,-,-,44,161311000
+2011-05,-,I,-,-,31.4,13701000
+2011-05,-,LA,-,-,41.6,49095000
+2011-05,-,MA,-,-,28.5,33908000
+2011-05,-,NA,-,-,39.5,82081000
+2011-05,-,W,-,-,29.9,411061000
+2011-06,-,AS,-,-,15.2,86539000
+2011-06,-,C,-,-,1.2,3960000
+2011-06,-,EU,-,-,41.7,153883000
+2011-06,-,I,-,-,32.1,14163000
+2011-06,-,LA,-,-,39.6,47203000
+2011-06,-,MA,-,-,25.8,30985000
+2011-06,-,NA,-,-,39.1,80753000
+2011-06,-,W,-,-,28.9,399362000
+2011-07,-,AS,-,-,15.7,90378000
+2011-07,-,C,-,-,1.5,4814000
+2011-07,-,EU,-,-,39.6,146724000
+2011-07,-,I,-,-,33.6,14979000
+2011-07,-,LA,-,-,35.4,42700000
+2011-07,-,MA,-,-,24.6,29882000
+2011-07,-,NA,-,-,40.5,83859000
+2011-07,-,W,-,-,28.2,393543000
+
+Ready
+
+
+Arguments
+ -i /a/wikistats/
+ -o /a/analytics
+
+Input folder: /a/wikistats/
+Output folder: /a/analytics
+
+Read '/a/wikistats//csv_wb/StatisticsMonthly.csv'
+Read '/a/wikistats//csv_wb/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats//csv_wk/StatisticsMonthly.csv'
+Read '/a/wikistats//csv_wk/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats//csv_wn/StatisticsMonthly.csv'
+Read '/a/wikistats//csv_wn/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats//csv_wp/StatisticsMonthly.csv'
+Read '/a/wikistats//csv_wp/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats//csv_wq/StatisticsMonthly.csv'
+Read '/a/wikistats//csv_wq/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats//csv_ws/StatisticsMonthly.csv'
+Read '/a/wikistats//csv_ws/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats//csv_wv/StatisticsMonthly.csv'
+Read '/a/wikistats//csv_wv/StatisticsUserActivitySpread.csv'
+Read '/a/wikistats//csv_wx/StatisticsMonthly.csv'
+Read '/a/wikistats//csv_wx/StatisticsUserActivitySpread.csv'
+Largest projects (most accumulated very active editors):
+wp,en,wp,de,wp,fr,wx,commons,wp,es,wp,ru,wp,ja,wp,it,wp,pl,wp,nl,wp,zh,
+
+No data yet for large wiki wp,ru for 2011-09-> skip month 2011-09
+No data yet for large wiki wp,fr for 2011-09-> skip month 2011-09
+No data yet for large wiki wp,nl for 2011-09-> skip month 2011-09
+No data yet for large wiki wp,ja for 2011-09-> skip month 2011-09
+No data yet for large wiki wp,zh for 2011-09-> skip month 2011-09
+No data yet for large wiki wp,pl for 2011-09-> skip month 2011-09
+No data yet for large wiki wp,it for 2011-09-> skip month 2011-09
+No data yet for large wiki wp,es for 2011-09-> skip month 2011-09
+No data yet for large wiki wx,commons for 2011-09-> skip month 2011-09
+No data yet for large wiki wp,en for 2011-09-> skip month 2011-09
+No data yet for large wiki wp,de for 2011-09-> skip month 2011-09

Added: trunk/wikistats/MetricsReporting/analytics/backup.sh
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/backup.sh (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/backup.sh 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+cd /a/wikistats/analytics
+
+dt=$(date +[%Y-%m-%d][%H:%M])
+
+zip /a/wikistats/backup/analytics_$dt.zip *.pl *.sh *.csv *.txt comscore/*.csv

Added: trunk/wikistats/MetricsReporting/analytics/use_db_analytics.txt
===================================================================
--- trunk/wikistats/MetricsReporting/analytics/use_db_analytics.txt (rev 0)
+++ trunk/wikistats/MetricsReporting/analytics/use_db_analytics.txt 2012-06-04 23:23:03 UTC (rev 115480)
@@ -0,0 +1,14 @@
+USE `analytics` ;
+
+SHOW TABLES ;
+
+SELECT project_code, SUM(editors_ge_5) AS editors FROM wikistats
+WHERE date >= '2011-04' AND date < '2011-05'
+GROUP BY project_code ;
+
+SELECT date, SUM(editors_ge_5) AS editors FROM wikistats
+WHERE date >= '2011-03' AND date < '2011-05'
+AND language_code <> 'commons'
+GROUP BY date ;
+
+


_______________________________________________
MediaWiki-CVS mailing list
MediaWiki-CVS [at] lists
https://lists.wikimedia.org/mailman/listinfo/mediawiki-cvs

Wikipedia mediawiki-cvs RSS feed   Index | Next | Previous | View Threaded
 
 


Interested in having your list archived? Contact Gossamer Threads
 
  Web Applications & Managed Hosting Powered by Gossamer Threads Inc.