Re: [Gnuplot-info] Help request - linear regression giving screwy results

[Thomas S. <t.s...@fz...>]

on the x-axis you have seconds since 2000-01-01, so your x-values
are around 315360000.
your y-values are around 20.

this leads to parameters which are different by at least 7 orders of
magnitude.

to bring the parameters into the same range, subtract 10 years and scale the
resulting x-values:

offset=10*365*24*60*60
f(x)=1.e-7*m*(x-offset)+b
m=7.
b=6.
fit f(x) 'comscore.dat' using 1:2 via m,b


Eric Raymond-3 wrote:
> 
> I've been attempting to do trend analysis on U.S. smartphone marketshare.
> You can see some of my successful visualizations at 
> 
>     http://www.catb.org/esr/comscore/
> 
> The problem is that I tried to do a linear-regression fit on Android's
> market share and got screwy results.  
> 
> To reproduce, save the enclosed comscore.dat and comscore-report, then
> do "comscore-report -p Android foo.png" and display foo.png.  What I
> see is that the linear-fit line passes through one datapoint but is
> nowhere
> near close to the right slope.  I don't get an error message.
> 
> Thanks in advance,
> -- 
> 		 http://www.catb.org/~esr/ Eric S. Raymond 
> 
> #!/usr/bin/env python
> """
> Generate reports from raw comScore market-share data
> 
> -s = tabulate or plot marketshare trends
> -u = tabulate or plot userbase trends
> -d = tabulate or plot changes in userbase by month
> -w = generate HTML table to stdout; without this, make a plot to a file
> -t = generate text table to stdout
> 
> The raw data is assumed to be in comscore.dat.
> """
> 
> import os, sys, getopt, tempfile, copy
> 
> class comScore:
>     def __init__(self, data):
>         self.data = data
>     #
>     # Framework code
>     #
>     def arithmetize(self):
>         "Turn data to numeric, excluding top row, left column, and -
> entries."
>         for i in range(1, len(self.data)):
>             for j in range(1, len(self.data[0])):
>                 if self.data[i][j] != '-':
>                     self.data[i][j] = float(self.data[i][j])
>     def unarithmetize(self):
>         "Turn numeric table data back to strings."
>         w = len(self.data[0])
>         d = len(self.data)
>         for i in range(1, d):
>             for j in range(1, w):
>                 if self.data[i][j] != '-':
>                     self.data[i][j] = "%.2f" % self.data[i][j]
>     def emit(self):
>         "Ship transformed self.data to a file for plotting."
>         (h, name) = tempfile.mkstemp()
>         ofp = open(name, "w")
>         d = len(self.data)
>         for i in range(d):
>             ofp.write("\t".join(self.data[i]) + "\n")
>         ofp.close()
>         return name
>     def textize(self, ofp=sys.stdout):
>         "Dump data as a tab-separated-values file."
>         for i in range(len(self.data)):
>             ofp.write("\t".join(self.data[i]) + "\n")
>     def webize(self, ofp=sys.stdout):
>         "Generate a table suitable for web display from specified
> self.data."
>         d = len(self.data)
>         w = len(self.data[0])
>         for i in range(d):
>             self.data[i][0] = self.data[i][0][:3] + " " +
> self.data[i][0][3:]
>         ofp.write("<table border='1'>\n")
>         for j in range(w):
>             ofp.write("<tr>")
>             for i in range(d):
>                 ofp.write("<td>" + self.data[i][j] + "</td>")
>             ofp.write("</tr>\n")
>         ofp.write("</table>\n")
>     def lastmonth(self):
>         "Return last month for which report is valid."
>         return self.data[len(self.data)-1][0]
>     def select(self, platform):
>         "Select out data for a single platform."
>         i = self.data[0].index(platform)
>         for j in range(len(self.data)):
>             self.data[j] = [self.data[j][0], self.data[j][i]]
>         self.data.pop(0)
>     #
>     # Data reduction
>     #
>     def usercount(self):
>         "Multiply market shares by smartphone userbase size (last
> column)."
>         w = len(self.data[0])
>         d = len(self.data)
>         self.data[0][w-1] = "Total"
>         for i in range(1, d):
>             for j in range(1, w-1):
>                 if self.data[i][j] != '-':
>                     self.data[i][j] *= self.data[i][w-1]
>                     self.data[i][j] /= 100.0
>     def deltas(self):
>         "Turn self.data into a differences table."
>         w = len(self.data[0])
>         d = len(self.data)
>         differences = copy.deepcopy(self.data)
>         for i in range(2, d):
>             for j in range(1, w):
>                 if self.data[i][j] == '-' or self.data[i-1][j] == '-':
>                     differences [i][j] = '-'
>                 else:
>                     differences [i][j] = self.data[i][j] -
> self.data[i-1][j]
>         # Remove first row, for which there is no corresponding delta.
>         self.data = differences[:1] + differences[2:]
> 
> coreplot = """
> set terminal png nocrop enhanced
> set output '%(output)s'
> set key outside right top vertical Right noreverse noenhanced autotitles
> nobox
> set datafile missing '-'
> set style data linespoints
> set xtics border in scale 1,0.5 nomirror rotate by -45 offset character 0,
> 0, 0
> set xtics norangelimit
> set xtics ()
> plot '%(input)s' using 2:xtic(1) title columnheader(2), \
>      		    '' using 3:xtic(1) title columnheader(3), \
>      		    '' using 4:xtic(1) title columnheader(4), \
>      		    '' using 5:xtic(1) title columnheader(5), \
>      		    '' using 6:xtic(1) title columnheader(6)
> """
> 
> predictive = """
> set terminal png nocrop enhanced
> set output '%(output)s'
> set datafile missing '-'
> set style data points
> set xtics border in scale 1,0.5 nomirror rotate by -45 offset character 0,
> 0, 0
> set xtics norangelimit
> set xtics ()
> set xdata time
> set timefmt '%%b%%Y'
> set xtics format '%%b%%Y'
> unset key
> f(x) = m*x + b
> fit f(x) '%(input)s' using 1:2 via m,b
> plot '%(input)s' using 1:2, f(x)
> """
> 
> def gnuplot(inputname, plot, outputname):
>     "Generate a derived plot."
>     plot = plot % {"output" : outputname, "input" : inputname}
>     print plot
>     ofp = os.popen("gnuplot -", "w")
>     ofp.write(plot)
>     ofp.close()
> 
> def grab(filename):
>     "Grab the contents of a data file."
>     lines = []
>     for line in open(filename):
>         if line[0] != '#':
>             lines.append(line.strip().split("\t"))
>     return lines
>  
> if __name__ == '__main__':
>     (options, arguments) = getopt.getopt(sys.argv[1:], "suwdtmp:")
>     plotprefix = ""
>     tabulate = False
>     textdump = False
>     datedump = False
>     share = False
>     user = False
>     deltas = False
>     predict = None
> 
>     basedata = grab('comscore.dat')
>     info = comScore(basedata)
> 
>     for (opt, val) in options:
>         if opt == '-s':
>             share = True
>         elif opt == '-u':
>             user = True
>         elif opt == '-d':
>             deltas = True
>         elif opt == '-w':
>             tabulate = True
>         elif opt == '-t':
>             textdump = True
>         elif opt == '-m':
>             datedump = True
>         elif opt == '-p':
>             predict = val
> 
>     if user:
>         title = "Userbase by platform, "
>     else:
>         title = "Market-share per platform, "
>     if deltas:
>         title += "change per month, "
>     if user:
>         yformat = "set format y '%%.0fM'\n"
>         title += "units of 1M users."
>     else:
>         yformat = "set format y '%%.0f%%%%'\n"
>         title += "units of 1%%."
>     title = "set title '%s'\n" % title
> 
>     if user or deltas:
>         info.arithmetize()
>         if user:
>             info.usercount()
>         if deltas:
>             info.deltas()
>         info.unarithmetize()
> 
>     if tabulate:
>         info.webize()
>     elif textdump:
>         info.textize()
>     elif datedump:
>         sys.stdout.write(info.lastmonth())
>     elif predict:
>         info.select(predict)
>         title = 'set title "%s"\n' % predict
>         gnuplot(info.emit(), title + yformat + predictive, arguments[0])
>     else:      
>         gnuplot(info.emit(), title + yformat + coreplot, arguments[0])
> 
> # End
> 
> ------------------------------------------------------------------------------
> Achieve unprecedented app performance and reliability
> What every C/C++ and Fortran developer should know.
> Learn how Intel has extended the reach of its next-generation tools
> to help boost performance applications - inlcuding clusters.
> http://p.sf.net/sfu/intel-dev2devmay
> _______________________________________________
> gnuplot-info mailing list
> gnu...@li...
> https://lists.sourceforge.net/lists/listinfo/gnuplot-info
> 
> 

-- 
View this message in context: http://old.nabble.com/Help-request---linear-regression-giving-screwy-results-tp31627742p31627998.html
Sent from the Gnuplot - User mailing list archive at Nabble.com.