Overfitting: an example using polynomial regression

Carl Bergstrom and Jevin West

Here’s a function that we want to try to fit. I’ve deliberately chosen one that is not a polynomial.

f[x_]:=30+70(1-4/(x+3));

Plot[f[x],{x,0,30},PlotRange->All]

We’ll want some gaussian noise. Here equal to a Gaussian with sigma=1/5 of the y value.

noise[y_]:=y Random[NormalDistribution[0,1]]/5

data=Table[{x,f[x]+noise[f[x]]},{x,0,10,1}]

restOfData=Table[{x,f[x]+noise[f[x]]},{x,10,20,1}]

linearfn[x_]:=Fit[data,{1,xx},xx]/.xx->x

linearfn[20]

quadfn[x_]:=Fit[data,{1,xx,xx^2},xx]/.xx->x

quadfn[20]

cubefn[x_]:=Fit[data,{1,xx,xx^2,xx^3},xx]/.xx->x

cubefn[20]

quartfn[x_]:=Fit[data,{1,xx,xx^2,xx^3,xx^4},xx]/.xx->x
quartfn[20]

quintfn[x_]:=Fit[data,{1,xx,xx^2,xx^3,xx^4,xx^5},xx]/.xx->x
quintfn[20]

Show[ListPlot[data,PlotStyle->{PointSize[0.02],ColorData[99,"ColorList"][[10]]},PlotRange->{{-1,21},{-30,205}},AxesOrigin->{0,0},ImageSize->1000,LabelStyle->{FontFamily->"Gill Sans",20,GrayLevel[0.3]},AxesLabel->{"kg Nitrogen","Yield"}],
Graphics[Text[Style["D",Large,Red],{20,Mean[data][[2]]}]],
Graphics[Text[Style["B",Large,Red],{20,linearfn[20]}]],
Graphics[Text[Style["E",Large,Red],{20,-20}]],
Graphics[Text[Style["A",Large,Red],{20,190}]],
Graphics[Text[Style["C",Large,Red],{20,100}]]]

Show[ListPlot[data,PlotStyle->{PointSize[0.02],ColorData[99,"ColorList"][[10]]},PlotRange->{{-1,21},{-30,205}},AxesOrigin->{0,0},ImageSize->1000,LabelStyle->{FontFamily->"Gill Sans",20,GrayLevel[0.3]},AxesLabel->{"kg Nitrogen","Yield"}],
Graphics[Text[Style["D",Large,Red],{20,Mean[data][[2]]}]],
Graphics[Text[Style["B",Large,Red],{20,linearfn[20]}]],
Graphics[Text[Style["E",Large,Red],{20,-20}]],
Graphics[Text[Style["A",Large,Red],{20,190}]],
Graphics[Text[Style["C",Large,Red],{20,100}]],
Plot[Mean[data][[2]],{x,0,40},PlotStyle->{Thickness[0.005],ColorData[99,"ColorList"][[11]]}]]

Show[ListPlot[data,PlotStyle->{PointSize[0.02],ColorData[99,"ColorList"][[10]]},PlotRange->{{-1,21},{-30,205}},AxesOrigin->{0,0},ImageSize->1000,LabelStyle->{FontFamily->"Gill Sans",20,GrayLevel[0.3]},AxesLabel->{"kg Nitrogen","Yield"}],
Graphics[Text[Style["D",Large,Red],{20,Mean[data][[2]]}]],
Graphics[Text[Style["B",Large,Red],{20,linearfn[20]}]],
Graphics[Text[Style["E",Large,Red],{20,-20}]],
Graphics[Text[Style["A",Large,Red],{20,190}]],
Graphics[Text[Style["C",Large,Red],{20,100}]],
Plot[linearfn[x],{x,0,40},PlotStyle->{Thickness[0.005],ColorData[99,"ColorList"][[11]]}]]

Show[ListPlot[data,PlotStyle->{PointSize[0.02],ColorData[99,"ColorList"][[10]]},PlotRange->{{-1,21},{-30,205}},AxesOrigin->{0,0},ImageSize->1000,LabelStyle->{FontFamily->"Gill Sans",20,GrayLevel[0.3]},AxesLabel->{"kg Nitrogen","Yield"}],
Graphics[Text[Style["D",Large,Red],{20,Mean[data][[2]]}]],
Graphics[Text[Style["B",Large,Red],{20,linearfn[20]}]],
Graphics[Text[Style["E",Large,Red],{20,-20}]],
Graphics[Text[Style["A",Large,Red],{20,190}]],
Graphics[Text[Style["C",Large,Red],{20,100}]],
Plot[quadfn[x],{x,0,40},PlotStyle->{Thickness[0.005],ColorData[99,"ColorList"][[11]]}]]

Show[ListPlot[data,PlotStyle->{PointSize[0.02],ColorData[99,"ColorList"][[10]]},PlotRange->{{-1,21},{-30,205}},AxesOrigin->{0,0},ImageSize->1000,LabelStyle->{FontFamily->"Gill Sans",20,GrayLevel[0.3]},AxesLabel->{"kg Nitrogen","Yield"}],
Graphics[Text[Style["D",Large,Red],{20,Mean[data][[2]]}]],
Graphics[Text[Style["B",Large,Red],{20,linearfn[20]}]],
Graphics[Text[Style["E",Large,Red],{20,-20}]],
Graphics[Text[Style["A",Large,Red],{20,190}]],
Graphics[Text[Style["C",Large,Red],{20,100}]],
Plot[cubefn[x],{x,0,40},PlotStyle->{Thickness[0.005],ColorData[99,"ColorList"][[11]]}]]

Show[ListPlot[data,PlotStyle->{PointSize[0.02],ColorData[99,"ColorList"][[10]]},PlotRange->{{-1,21},{0,150}},AxesOrigin->{0,0},ImageSize->600,LabelStyle->{FontFamily->"Gill Sans",14,GrayLevel[0.3]}],
ListPlot[restOfData,PlotStyle->{PointSize[0.02],ColorData[99,"ColorList"][[9]]},PlotRange->{{-1,21},{0,150}},AxesOrigin->{0,0},ImageSize->600,LabelStyle->{FontFamily->"Gill Sans",14,GrayLevel[0.3]}],
Plot[Mean[data][[2]],{x,0,40},PlotStyle->{Thickness[0.005],ColorData[99,"ColorList"][[11]]}]]

linearfn[x_]:=Fit[data,{1,xx},xx]/.xx->x

linear

linearfn[20]

Show[ListPlot[data,PlotStyle->{PointSize[0.02],ColorData[99,"ColorList"][[10]]},PlotRange->{{-1,21},{0,150}},AxesOrigin->{0,0},ImageSize->600,LabelStyle->{FontFamily->"Gill Sans",14,GrayLevel[0.3]}],
ListPlot[restOfData,PlotStyle->{PointSize[0.02],ColorData[99,"ColorList"][[9]]},PlotRange->{{-1,21},{0,150}},AxesOrigin->{0,0},ImageSize->600,LabelStyle->{FontFamily->"Gill Sans",14,GrayLevel[0.3]}],
Plot[Mean[data][[2]],{x,0,40},PlotStyle->{Thickness[0.005],ColorData[99,"ColorList"][[11]]}]]

Show[ListPlot[Labeled[data,PlotStyle->{PointSize[0.02],ColorData[99,"ColorList"][[10]]},PlotRange->{{-1,50},{0,150}},AxesOrigin->{0,0},ImageSize->600,LabelStyle->{FontFamily->"Gill Sans",14,GrayLevel[0.3]}],
Plot[Mean[data][[2]],{x,0,50},PlotStyle->{Thickness[0.005],ColorData[99,"ColorList"][[11]]}]]