diff --git a/09-29/myscript.ipynb b/Lessons/09-29/myscript.ipynb similarity index 100% rename from 09-29/myscript.ipynb rename to Lessons/09-29/myscript.ipynb diff --git a/09-29/salaries.csv b/Lessons/09-29/salaries.csv similarity index 100% rename from 09-29/salaries.csv rename to Lessons/09-29/salaries.csv diff --git a/10-04/GMQ.ipynb b/Lessons/10-04/GMQ.ipynb similarity index 100% rename from 10-04/GMQ.ipynb rename to Lessons/10-04/GMQ.ipynb diff --git a/10-04/GMQ.jl b/Lessons/10-04/GMQ.jl similarity index 100% rename from 10-04/GMQ.jl rename to Lessons/10-04/GMQ.jl diff --git a/10-04/GMQ.m b/Lessons/10-04/GMQ.m similarity index 100% rename from 10-04/GMQ.m rename to Lessons/10-04/GMQ.m diff --git a/10-04/genQf.jl b/Lessons/10-04/genQf.jl similarity index 100% rename from 10-04/genQf.jl rename to Lessons/10-04/genQf.jl diff --git a/10-04/getQf.m b/Lessons/10-04/getQf.m similarity index 100% rename from 10-04/getQf.m rename to Lessons/10-04/getQf.m diff --git a/10-04/lesson.ipynb b/Lessons/10-04/lesson.ipynb similarity index 100% rename from 10-04/lesson.ipynb rename to Lessons/10-04/lesson.ipynb diff --git a/10-04/optQf.m b/Lessons/10-04/optQf.m similarity index 100% rename from 10-04/optQf.m rename to Lessons/10-04/optQf.m diff --git a/10-04/plotQ.jl b/Lessons/10-04/plotQ.jl similarity index 100% rename from 10-04/plotQ.jl rename to Lessons/10-04/plotQ.jl diff --git a/10-04/plotQ.m b/Lessons/10-04/plotQ.m similarity index 100% rename from 10-04/plotQ.m rename to Lessons/10-04/plotQ.m diff --git a/10-12/DIS.jl b/Lessons/10-12/DIS.jl similarity index 100% rename from 10-12/DIS.jl rename to Lessons/10-12/DIS.jl diff --git a/10-12/DIS.m b/Lessons/10-12/DIS.m similarity index 100% rename from 10-12/DIS.m rename to Lessons/10-12/DIS.m diff --git a/10-12/GRS.jl b/Lessons/10-12/GRS.jl similarity index 100% rename from 10-12/GRS.jl rename to Lessons/10-12/GRS.jl diff --git a/10-12/GRS.m b/Lessons/10-12/GRS.m similarity index 100% rename from 10-12/GRS.m rename to Lessons/10-12/GRS.m diff --git a/10-12/UNM.jl b/Lessons/10-12/UNM.jl similarity index 100% rename from 10-12/UNM.jl rename to Lessons/10-12/UNM.jl diff --git a/10-12/UNM.m b/Lessons/10-12/UNM.m similarity index 100% rename from 10-12/UNM.m rename to Lessons/10-12/UNM.m diff --git a/10-12/lesson.ipynb b/Lessons/10-12/lesson.ipynb similarity index 100% rename from 10-12/lesson.ipynb rename to Lessons/10-12/lesson.ipynb diff --git a/10-12/oneDTestFunctions.jl b/Lessons/10-12/oneDTestFunctions.jl similarity index 100% rename from 10-12/oneDTestFunctions.jl rename to Lessons/10-12/oneDTestFunctions.jl diff --git a/10-12/oneDTestFunctions.m b/Lessons/10-12/oneDTestFunctions.m similarity index 100% rename from 10-12/oneDTestFunctions.m rename to Lessons/10-12/oneDTestFunctions.m diff --git a/10-18/lesson.ipynb b/Lessons/10-18/lesson.ipynb similarity index 100% rename from 10-18/lesson.ipynb rename to Lessons/10-18/lesson.ipynb diff --git a/10-20/lesson.ipynb b/Lessons/10-20/lesson.ipynb similarity index 100% rename from 10-20/lesson.ipynb rename to Lessons/10-20/lesson.ipynb diff --git a/10-25/lesson.ipynb b/Lessons/10-25/lesson.ipynb similarity index 100% rename from 10-25/lesson.ipynb rename to Lessons/10-25/lesson.ipynb diff --git a/10-26/lesson.ipynb b/Lessons/10-26/lesson.ipynb similarity index 100% rename from 10-26/lesson.ipynb rename to Lessons/10-26/lesson.ipynb diff --git a/10-27/lesson.ipynb b/Lessons/10-27/lesson.ipynb similarity index 100% rename from 10-27/lesson.ipynb rename to Lessons/10-27/lesson.ipynb diff --git a/11-03/Katharine_Hepburn_publicity_photograph.jpg b/Lessons/11-03/Katharine_Hepburn_publicity_photograph.jpg similarity index 100% rename from 11-03/Katharine_Hepburn_publicity_photograph.jpg rename to Lessons/11-03/Katharine_Hepburn_publicity_photograph.jpg diff --git a/11-03/cameraman.tif b/Lessons/11-03/cameraman.tif similarity index 100% rename from 11-03/cameraman.tif rename to Lessons/11-03/cameraman.tif diff --git a/11-03/eigenfaces/eigenfaces_classify.m b/Lessons/11-03/eigenfaces/eigenfaces_classify.m similarity index 100% rename from 11-03/eigenfaces/eigenfaces_classify.m rename to Lessons/11-03/eigenfaces/eigenfaces_classify.m diff --git a/11-03/eigenfaces/eigenfaces_scatter.m b/Lessons/11-03/eigenfaces/eigenfaces_scatter.m similarity index 100% rename from 11-03/eigenfaces/eigenfaces_scatter.m rename to Lessons/11-03/eigenfaces/eigenfaces_scatter.m diff --git a/11-03/eigenfaces/interactiverec.m b/Lessons/11-03/eigenfaces/interactiverec.m similarity index 100% rename from 11-03/eigenfaces/interactiverec.m rename to Lessons/11-03/eigenfaces/interactiverec.m diff --git a/11-03/eigenfaces/readyalefaces_to_tensor.m b/Lessons/11-03/eigenfaces/readyalefaces_to_tensor.m similarity index 100% rename from 11-03/eigenfaces/readyalefaces_to_tensor.m rename to Lessons/11-03/eigenfaces/readyalefaces_to_tensor.m diff --git a/11-03/eigenfaces/showyalefaces.m b/Lessons/11-03/eigenfaces/showyalefaces.m similarity index 100% rename from 11-03/eigenfaces/showyalefaces.m rename to Lessons/11-03/eigenfaces/showyalefaces.m diff --git a/11-03/eigenfaces/tightsubplot.m b/Lessons/11-03/eigenfaces/tightsubplot.m similarity index 100% rename from 11-03/eigenfaces/tightsubplot.m rename to Lessons/11-03/eigenfaces/tightsubplot.m diff --git a/11-03/lesson.ipynb b/Lessons/11-03/lesson.ipynb similarity index 100% rename from 11-03/lesson.ipynb rename to Lessons/11-03/lesson.ipynb diff --git a/11-03/otherfaces/bart.png b/Lessons/11-03/otherfaces/bart.png similarity index 100% rename from 11-03/otherfaces/bart.png rename to Lessons/11-03/otherfaces/bart.png diff --git a/11-03/otherfaces/bart.xcf b/Lessons/11-03/otherfaces/bart.xcf similarity index 100% rename from 11-03/otherfaces/bart.xcf rename to Lessons/11-03/otherfaces/bart.xcf diff --git a/11-03/otherfaces/bart_unedited.jpg b/Lessons/11-03/otherfaces/bart_unedited.jpg similarity index 100% rename from 11-03/otherfaces/bart_unedited.jpg rename to Lessons/11-03/otherfaces/bart_unedited.jpg diff --git a/11-03/otherfaces/car.png b/Lessons/11-03/otherfaces/car.png similarity index 100% rename from 11-03/otherfaces/car.png rename to Lessons/11-03/otherfaces/car.png diff --git a/11-03/otherfaces/car.xcf b/Lessons/11-03/otherfaces/car.xcf similarity index 100% rename from 11-03/otherfaces/car.xcf rename to Lessons/11-03/otherfaces/car.xcf diff --git a/11-03/otherfaces/car_unedited.jpg b/Lessons/11-03/otherfaces/car_unedited.jpg similarity index 100% rename from 11-03/otherfaces/car_unedited.jpg rename to Lessons/11-03/otherfaces/car_unedited.jpg diff --git a/11-03/readyalefaces_to_tensor.jl b/Lessons/11-03/readyalefaces_to_tensor.jl similarity index 100% rename from 11-03/readyalefaces_to_tensor.jl rename to Lessons/11-03/readyalefaces_to_tensor.jl diff --git a/11-03/showyalefaces.jl b/Lessons/11-03/showyalefaces.jl similarity index 100% rename from 11-03/showyalefaces.jl rename to Lessons/11-03/showyalefaces.jl diff --git a/11-03/yalefaces/README b/Lessons/11-03/yalefaces/README similarity index 100% rename from 11-03/yalefaces/README rename to Lessons/11-03/yalefaces/README diff --git a/11-03/yalefaces/subject01.centerlight.gif b/Lessons/11-03/yalefaces/subject01.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject01.centerlight.gif rename to Lessons/11-03/yalefaces/subject01.centerlight.gif diff --git a/11-03/yalefaces/subject01.glasses.gif b/Lessons/11-03/yalefaces/subject01.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject01.glasses.gif rename to Lessons/11-03/yalefaces/subject01.glasses.gif diff --git a/11-03/yalefaces/subject01.happy.gif b/Lessons/11-03/yalefaces/subject01.happy.gif similarity index 100% rename from 11-03/yalefaces/subject01.happy.gif rename to Lessons/11-03/yalefaces/subject01.happy.gif diff --git a/11-03/yalefaces/subject01.leftlight.gif b/Lessons/11-03/yalefaces/subject01.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject01.leftlight.gif rename to Lessons/11-03/yalefaces/subject01.leftlight.gif diff --git a/11-03/yalefaces/subject01.noglasses.gif b/Lessons/11-03/yalefaces/subject01.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject01.noglasses.gif rename to Lessons/11-03/yalefaces/subject01.noglasses.gif diff --git a/11-03/yalefaces/subject01.normal.gif b/Lessons/11-03/yalefaces/subject01.normal.gif similarity index 100% rename from 11-03/yalefaces/subject01.normal.gif rename to Lessons/11-03/yalefaces/subject01.normal.gif diff --git a/11-03/yalefaces/subject01.rightlight.gif b/Lessons/11-03/yalefaces/subject01.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject01.rightlight.gif rename to Lessons/11-03/yalefaces/subject01.rightlight.gif diff --git a/11-03/yalefaces/subject01.sad.gif b/Lessons/11-03/yalefaces/subject01.sad.gif similarity index 100% rename from 11-03/yalefaces/subject01.sad.gif rename to Lessons/11-03/yalefaces/subject01.sad.gif diff --git a/11-03/yalefaces/subject01.sleepy.gif b/Lessons/11-03/yalefaces/subject01.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject01.sleepy.gif rename to Lessons/11-03/yalefaces/subject01.sleepy.gif diff --git a/11-03/yalefaces/subject01.surprised.gif b/Lessons/11-03/yalefaces/subject01.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject01.surprised.gif rename to Lessons/11-03/yalefaces/subject01.surprised.gif diff --git a/11-03/yalefaces/subject01.wink.gif b/Lessons/11-03/yalefaces/subject01.wink.gif similarity index 100% rename from 11-03/yalefaces/subject01.wink.gif rename to Lessons/11-03/yalefaces/subject01.wink.gif diff --git a/11-03/yalefaces/subject02.centerlight.gif b/Lessons/11-03/yalefaces/subject02.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject02.centerlight.gif rename to Lessons/11-03/yalefaces/subject02.centerlight.gif diff --git a/11-03/yalefaces/subject02.glasses.gif b/Lessons/11-03/yalefaces/subject02.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject02.glasses.gif rename to Lessons/11-03/yalefaces/subject02.glasses.gif diff --git a/11-03/yalefaces/subject02.happy.gif b/Lessons/11-03/yalefaces/subject02.happy.gif similarity index 100% rename from 11-03/yalefaces/subject02.happy.gif rename to Lessons/11-03/yalefaces/subject02.happy.gif diff --git a/11-03/yalefaces/subject02.leftlight.gif b/Lessons/11-03/yalefaces/subject02.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject02.leftlight.gif rename to Lessons/11-03/yalefaces/subject02.leftlight.gif diff --git a/11-03/yalefaces/subject02.noglasses.gif b/Lessons/11-03/yalefaces/subject02.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject02.noglasses.gif rename to Lessons/11-03/yalefaces/subject02.noglasses.gif diff --git a/11-03/yalefaces/subject02.normal.gif b/Lessons/11-03/yalefaces/subject02.normal.gif similarity index 100% rename from 11-03/yalefaces/subject02.normal.gif rename to Lessons/11-03/yalefaces/subject02.normal.gif diff --git a/11-03/yalefaces/subject02.rightlight.gif b/Lessons/11-03/yalefaces/subject02.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject02.rightlight.gif rename to Lessons/11-03/yalefaces/subject02.rightlight.gif diff --git a/11-03/yalefaces/subject02.sad.gif b/Lessons/11-03/yalefaces/subject02.sad.gif similarity index 100% rename from 11-03/yalefaces/subject02.sad.gif rename to Lessons/11-03/yalefaces/subject02.sad.gif diff --git a/11-03/yalefaces/subject02.sleepy.gif b/Lessons/11-03/yalefaces/subject02.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject02.sleepy.gif rename to Lessons/11-03/yalefaces/subject02.sleepy.gif diff --git a/11-03/yalefaces/subject02.surprised.gif b/Lessons/11-03/yalefaces/subject02.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject02.surprised.gif rename to Lessons/11-03/yalefaces/subject02.surprised.gif diff --git a/11-03/yalefaces/subject02.wink.gif b/Lessons/11-03/yalefaces/subject02.wink.gif similarity index 100% rename from 11-03/yalefaces/subject02.wink.gif rename to Lessons/11-03/yalefaces/subject02.wink.gif diff --git a/11-03/yalefaces/subject03.centerlight.gif b/Lessons/11-03/yalefaces/subject03.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject03.centerlight.gif rename to Lessons/11-03/yalefaces/subject03.centerlight.gif diff --git a/11-03/yalefaces/subject03.glasses.gif b/Lessons/11-03/yalefaces/subject03.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject03.glasses.gif rename to Lessons/11-03/yalefaces/subject03.glasses.gif diff --git a/11-03/yalefaces/subject03.happy.gif b/Lessons/11-03/yalefaces/subject03.happy.gif similarity index 100% rename from 11-03/yalefaces/subject03.happy.gif rename to Lessons/11-03/yalefaces/subject03.happy.gif diff --git a/11-03/yalefaces/subject03.leftlight.gif b/Lessons/11-03/yalefaces/subject03.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject03.leftlight.gif rename to Lessons/11-03/yalefaces/subject03.leftlight.gif diff --git a/11-03/yalefaces/subject03.noglasses.gif b/Lessons/11-03/yalefaces/subject03.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject03.noglasses.gif rename to Lessons/11-03/yalefaces/subject03.noglasses.gif diff --git a/11-03/yalefaces/subject03.normal.gif b/Lessons/11-03/yalefaces/subject03.normal.gif similarity index 100% rename from 11-03/yalefaces/subject03.normal.gif rename to Lessons/11-03/yalefaces/subject03.normal.gif diff --git a/11-03/yalefaces/subject03.rightlight.gif b/Lessons/11-03/yalefaces/subject03.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject03.rightlight.gif rename to Lessons/11-03/yalefaces/subject03.rightlight.gif diff --git a/11-03/yalefaces/subject03.sad.gif b/Lessons/11-03/yalefaces/subject03.sad.gif similarity index 100% rename from 11-03/yalefaces/subject03.sad.gif rename to Lessons/11-03/yalefaces/subject03.sad.gif diff --git a/11-03/yalefaces/subject03.sleepy.gif b/Lessons/11-03/yalefaces/subject03.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject03.sleepy.gif rename to Lessons/11-03/yalefaces/subject03.sleepy.gif diff --git a/11-03/yalefaces/subject03.surprised.gif b/Lessons/11-03/yalefaces/subject03.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject03.surprised.gif rename to Lessons/11-03/yalefaces/subject03.surprised.gif diff --git a/11-03/yalefaces/subject03.wink.gif b/Lessons/11-03/yalefaces/subject03.wink.gif similarity index 100% rename from 11-03/yalefaces/subject03.wink.gif rename to Lessons/11-03/yalefaces/subject03.wink.gif diff --git a/11-03/yalefaces/subject04.centerlight.gif b/Lessons/11-03/yalefaces/subject04.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject04.centerlight.gif rename to Lessons/11-03/yalefaces/subject04.centerlight.gif diff --git a/11-03/yalefaces/subject04.glasses.gif b/Lessons/11-03/yalefaces/subject04.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject04.glasses.gif rename to Lessons/11-03/yalefaces/subject04.glasses.gif diff --git a/11-03/yalefaces/subject04.happy.gif b/Lessons/11-03/yalefaces/subject04.happy.gif similarity index 100% rename from 11-03/yalefaces/subject04.happy.gif rename to Lessons/11-03/yalefaces/subject04.happy.gif diff --git a/11-03/yalefaces/subject04.leftlight.gif b/Lessons/11-03/yalefaces/subject04.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject04.leftlight.gif rename to Lessons/11-03/yalefaces/subject04.leftlight.gif diff --git a/11-03/yalefaces/subject04.noglasses.gif b/Lessons/11-03/yalefaces/subject04.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject04.noglasses.gif rename to Lessons/11-03/yalefaces/subject04.noglasses.gif diff --git a/11-03/yalefaces/subject04.normal.gif b/Lessons/11-03/yalefaces/subject04.normal.gif similarity index 100% rename from 11-03/yalefaces/subject04.normal.gif rename to Lessons/11-03/yalefaces/subject04.normal.gif diff --git a/11-03/yalefaces/subject04.rightlight.gif b/Lessons/11-03/yalefaces/subject04.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject04.rightlight.gif rename to Lessons/11-03/yalefaces/subject04.rightlight.gif diff --git a/11-03/yalefaces/subject04.sad.gif b/Lessons/11-03/yalefaces/subject04.sad.gif similarity index 100% rename from 11-03/yalefaces/subject04.sad.gif rename to Lessons/11-03/yalefaces/subject04.sad.gif diff --git a/11-03/yalefaces/subject04.sleepy.gif b/Lessons/11-03/yalefaces/subject04.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject04.sleepy.gif rename to Lessons/11-03/yalefaces/subject04.sleepy.gif diff --git a/11-03/yalefaces/subject04.surprised.gif b/Lessons/11-03/yalefaces/subject04.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject04.surprised.gif rename to Lessons/11-03/yalefaces/subject04.surprised.gif diff --git a/11-03/yalefaces/subject04.wink.gif b/Lessons/11-03/yalefaces/subject04.wink.gif similarity index 100% rename from 11-03/yalefaces/subject04.wink.gif rename to Lessons/11-03/yalefaces/subject04.wink.gif diff --git a/11-03/yalefaces/subject05.centerlight.gif b/Lessons/11-03/yalefaces/subject05.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject05.centerlight.gif rename to Lessons/11-03/yalefaces/subject05.centerlight.gif diff --git a/11-03/yalefaces/subject05.glasses.gif b/Lessons/11-03/yalefaces/subject05.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject05.glasses.gif rename to Lessons/11-03/yalefaces/subject05.glasses.gif diff --git a/11-03/yalefaces/subject05.happy.gif b/Lessons/11-03/yalefaces/subject05.happy.gif similarity index 100% rename from 11-03/yalefaces/subject05.happy.gif rename to Lessons/11-03/yalefaces/subject05.happy.gif diff --git a/11-03/yalefaces/subject05.leftlight.gif b/Lessons/11-03/yalefaces/subject05.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject05.leftlight.gif rename to Lessons/11-03/yalefaces/subject05.leftlight.gif diff --git a/11-03/yalefaces/subject05.noglasses.gif b/Lessons/11-03/yalefaces/subject05.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject05.noglasses.gif rename to Lessons/11-03/yalefaces/subject05.noglasses.gif diff --git a/11-03/yalefaces/subject05.normal.gif b/Lessons/11-03/yalefaces/subject05.normal.gif similarity index 100% rename from 11-03/yalefaces/subject05.normal.gif rename to Lessons/11-03/yalefaces/subject05.normal.gif diff --git a/11-03/yalefaces/subject05.rightlight.gif b/Lessons/11-03/yalefaces/subject05.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject05.rightlight.gif rename to Lessons/11-03/yalefaces/subject05.rightlight.gif diff --git a/11-03/yalefaces/subject05.sad.gif b/Lessons/11-03/yalefaces/subject05.sad.gif similarity index 100% rename from 11-03/yalefaces/subject05.sad.gif rename to Lessons/11-03/yalefaces/subject05.sad.gif diff --git a/11-03/yalefaces/subject05.sleepy.gif b/Lessons/11-03/yalefaces/subject05.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject05.sleepy.gif rename to Lessons/11-03/yalefaces/subject05.sleepy.gif diff --git a/11-03/yalefaces/subject05.surprised.gif b/Lessons/11-03/yalefaces/subject05.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject05.surprised.gif rename to Lessons/11-03/yalefaces/subject05.surprised.gif diff --git a/11-03/yalefaces/subject05.wink.gif b/Lessons/11-03/yalefaces/subject05.wink.gif similarity index 100% rename from 11-03/yalefaces/subject05.wink.gif rename to Lessons/11-03/yalefaces/subject05.wink.gif diff --git a/11-03/yalefaces/subject06.centerlight.gif b/Lessons/11-03/yalefaces/subject06.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject06.centerlight.gif rename to Lessons/11-03/yalefaces/subject06.centerlight.gif diff --git a/11-03/yalefaces/subject06.glasses.gif b/Lessons/11-03/yalefaces/subject06.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject06.glasses.gif rename to Lessons/11-03/yalefaces/subject06.glasses.gif diff --git a/11-03/yalefaces/subject06.happy.gif b/Lessons/11-03/yalefaces/subject06.happy.gif similarity index 100% rename from 11-03/yalefaces/subject06.happy.gif rename to Lessons/11-03/yalefaces/subject06.happy.gif diff --git a/11-03/yalefaces/subject06.leftlight.gif b/Lessons/11-03/yalefaces/subject06.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject06.leftlight.gif rename to Lessons/11-03/yalefaces/subject06.leftlight.gif diff --git a/11-03/yalefaces/subject06.noglasses.gif b/Lessons/11-03/yalefaces/subject06.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject06.noglasses.gif rename to Lessons/11-03/yalefaces/subject06.noglasses.gif diff --git a/11-03/yalefaces/subject06.normal.gif b/Lessons/11-03/yalefaces/subject06.normal.gif similarity index 100% rename from 11-03/yalefaces/subject06.normal.gif rename to Lessons/11-03/yalefaces/subject06.normal.gif diff --git a/11-03/yalefaces/subject06.rightlight.gif b/Lessons/11-03/yalefaces/subject06.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject06.rightlight.gif rename to Lessons/11-03/yalefaces/subject06.rightlight.gif diff --git a/11-03/yalefaces/subject06.sad.gif b/Lessons/11-03/yalefaces/subject06.sad.gif similarity index 100% rename from 11-03/yalefaces/subject06.sad.gif rename to Lessons/11-03/yalefaces/subject06.sad.gif diff --git a/11-03/yalefaces/subject06.sleepy.gif b/Lessons/11-03/yalefaces/subject06.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject06.sleepy.gif rename to Lessons/11-03/yalefaces/subject06.sleepy.gif diff --git a/11-03/yalefaces/subject06.surprised.gif b/Lessons/11-03/yalefaces/subject06.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject06.surprised.gif rename to Lessons/11-03/yalefaces/subject06.surprised.gif diff --git a/11-03/yalefaces/subject06.wink.gif b/Lessons/11-03/yalefaces/subject06.wink.gif similarity index 100% rename from 11-03/yalefaces/subject06.wink.gif rename to Lessons/11-03/yalefaces/subject06.wink.gif diff --git a/11-03/yalefaces/subject07.centerlight.gif b/Lessons/11-03/yalefaces/subject07.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject07.centerlight.gif rename to Lessons/11-03/yalefaces/subject07.centerlight.gif diff --git a/11-03/yalefaces/subject07.glasses.gif b/Lessons/11-03/yalefaces/subject07.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject07.glasses.gif rename to Lessons/11-03/yalefaces/subject07.glasses.gif diff --git a/11-03/yalefaces/subject07.happy.gif b/Lessons/11-03/yalefaces/subject07.happy.gif similarity index 100% rename from 11-03/yalefaces/subject07.happy.gif rename to Lessons/11-03/yalefaces/subject07.happy.gif diff --git a/11-03/yalefaces/subject07.leftlight.gif b/Lessons/11-03/yalefaces/subject07.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject07.leftlight.gif rename to Lessons/11-03/yalefaces/subject07.leftlight.gif diff --git a/11-03/yalefaces/subject07.noglasses.gif b/Lessons/11-03/yalefaces/subject07.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject07.noglasses.gif rename to Lessons/11-03/yalefaces/subject07.noglasses.gif diff --git a/11-03/yalefaces/subject07.normal.gif b/Lessons/11-03/yalefaces/subject07.normal.gif similarity index 100% rename from 11-03/yalefaces/subject07.normal.gif rename to Lessons/11-03/yalefaces/subject07.normal.gif diff --git a/11-03/yalefaces/subject07.rightlight.gif b/Lessons/11-03/yalefaces/subject07.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject07.rightlight.gif rename to Lessons/11-03/yalefaces/subject07.rightlight.gif diff --git a/11-03/yalefaces/subject07.sad.gif b/Lessons/11-03/yalefaces/subject07.sad.gif similarity index 100% rename from 11-03/yalefaces/subject07.sad.gif rename to Lessons/11-03/yalefaces/subject07.sad.gif diff --git a/11-03/yalefaces/subject07.sleepy.gif b/Lessons/11-03/yalefaces/subject07.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject07.sleepy.gif rename to Lessons/11-03/yalefaces/subject07.sleepy.gif diff --git a/11-03/yalefaces/subject07.surprised.gif b/Lessons/11-03/yalefaces/subject07.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject07.surprised.gif rename to Lessons/11-03/yalefaces/subject07.surprised.gif diff --git a/11-03/yalefaces/subject07.wink.gif b/Lessons/11-03/yalefaces/subject07.wink.gif similarity index 100% rename from 11-03/yalefaces/subject07.wink.gif rename to Lessons/11-03/yalefaces/subject07.wink.gif diff --git a/11-03/yalefaces/subject08.centerlight.gif b/Lessons/11-03/yalefaces/subject08.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject08.centerlight.gif rename to Lessons/11-03/yalefaces/subject08.centerlight.gif diff --git a/11-03/yalefaces/subject08.glasses.gif b/Lessons/11-03/yalefaces/subject08.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject08.glasses.gif rename to Lessons/11-03/yalefaces/subject08.glasses.gif diff --git a/11-03/yalefaces/subject08.happy.gif b/Lessons/11-03/yalefaces/subject08.happy.gif similarity index 100% rename from 11-03/yalefaces/subject08.happy.gif rename to Lessons/11-03/yalefaces/subject08.happy.gif diff --git a/11-03/yalefaces/subject08.leftlight.gif b/Lessons/11-03/yalefaces/subject08.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject08.leftlight.gif rename to Lessons/11-03/yalefaces/subject08.leftlight.gif diff --git a/11-03/yalefaces/subject08.noglasses.gif b/Lessons/11-03/yalefaces/subject08.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject08.noglasses.gif rename to Lessons/11-03/yalefaces/subject08.noglasses.gif diff --git a/11-03/yalefaces/subject08.normal.gif b/Lessons/11-03/yalefaces/subject08.normal.gif similarity index 100% rename from 11-03/yalefaces/subject08.normal.gif rename to Lessons/11-03/yalefaces/subject08.normal.gif diff --git a/11-03/yalefaces/subject08.rightlight.gif b/Lessons/11-03/yalefaces/subject08.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject08.rightlight.gif rename to Lessons/11-03/yalefaces/subject08.rightlight.gif diff --git a/11-03/yalefaces/subject08.sad.gif b/Lessons/11-03/yalefaces/subject08.sad.gif similarity index 100% rename from 11-03/yalefaces/subject08.sad.gif rename to Lessons/11-03/yalefaces/subject08.sad.gif diff --git a/11-03/yalefaces/subject08.sleepy.gif b/Lessons/11-03/yalefaces/subject08.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject08.sleepy.gif rename to Lessons/11-03/yalefaces/subject08.sleepy.gif diff --git a/11-03/yalefaces/subject08.surprised.gif b/Lessons/11-03/yalefaces/subject08.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject08.surprised.gif rename to Lessons/11-03/yalefaces/subject08.surprised.gif diff --git a/11-03/yalefaces/subject08.wink.gif b/Lessons/11-03/yalefaces/subject08.wink.gif similarity index 100% rename from 11-03/yalefaces/subject08.wink.gif rename to Lessons/11-03/yalefaces/subject08.wink.gif diff --git a/11-03/yalefaces/subject09.centerlight.gif b/Lessons/11-03/yalefaces/subject09.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject09.centerlight.gif rename to Lessons/11-03/yalefaces/subject09.centerlight.gif diff --git a/11-03/yalefaces/subject09.glasses.gif b/Lessons/11-03/yalefaces/subject09.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject09.glasses.gif rename to Lessons/11-03/yalefaces/subject09.glasses.gif diff --git a/11-03/yalefaces/subject09.happy.gif b/Lessons/11-03/yalefaces/subject09.happy.gif similarity index 100% rename from 11-03/yalefaces/subject09.happy.gif rename to Lessons/11-03/yalefaces/subject09.happy.gif diff --git a/11-03/yalefaces/subject09.leftlight.gif b/Lessons/11-03/yalefaces/subject09.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject09.leftlight.gif rename to Lessons/11-03/yalefaces/subject09.leftlight.gif diff --git a/11-03/yalefaces/subject09.noglasses.gif b/Lessons/11-03/yalefaces/subject09.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject09.noglasses.gif rename to Lessons/11-03/yalefaces/subject09.noglasses.gif diff --git a/11-03/yalefaces/subject09.normal.gif b/Lessons/11-03/yalefaces/subject09.normal.gif similarity index 100% rename from 11-03/yalefaces/subject09.normal.gif rename to Lessons/11-03/yalefaces/subject09.normal.gif diff --git a/11-03/yalefaces/subject09.rightlight.gif b/Lessons/11-03/yalefaces/subject09.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject09.rightlight.gif rename to Lessons/11-03/yalefaces/subject09.rightlight.gif diff --git a/11-03/yalefaces/subject09.sad.gif b/Lessons/11-03/yalefaces/subject09.sad.gif similarity index 100% rename from 11-03/yalefaces/subject09.sad.gif rename to Lessons/11-03/yalefaces/subject09.sad.gif diff --git a/11-03/yalefaces/subject09.sleepy.gif b/Lessons/11-03/yalefaces/subject09.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject09.sleepy.gif rename to Lessons/11-03/yalefaces/subject09.sleepy.gif diff --git a/11-03/yalefaces/subject09.surprised.gif b/Lessons/11-03/yalefaces/subject09.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject09.surprised.gif rename to Lessons/11-03/yalefaces/subject09.surprised.gif diff --git a/11-03/yalefaces/subject09.wink.gif b/Lessons/11-03/yalefaces/subject09.wink.gif similarity index 100% rename from 11-03/yalefaces/subject09.wink.gif rename to Lessons/11-03/yalefaces/subject09.wink.gif diff --git a/11-03/yalefaces/subject10.centerlight.gif b/Lessons/11-03/yalefaces/subject10.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject10.centerlight.gif rename to Lessons/11-03/yalefaces/subject10.centerlight.gif diff --git a/11-03/yalefaces/subject10.glasses.gif b/Lessons/11-03/yalefaces/subject10.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject10.glasses.gif rename to Lessons/11-03/yalefaces/subject10.glasses.gif diff --git a/11-03/yalefaces/subject10.happy.gif b/Lessons/11-03/yalefaces/subject10.happy.gif similarity index 100% rename from 11-03/yalefaces/subject10.happy.gif rename to Lessons/11-03/yalefaces/subject10.happy.gif diff --git a/11-03/yalefaces/subject10.leftlight.gif b/Lessons/11-03/yalefaces/subject10.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject10.leftlight.gif rename to Lessons/11-03/yalefaces/subject10.leftlight.gif diff --git a/11-03/yalefaces/subject10.noglasses.gif b/Lessons/11-03/yalefaces/subject10.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject10.noglasses.gif rename to Lessons/11-03/yalefaces/subject10.noglasses.gif diff --git a/11-03/yalefaces/subject10.normal.gif b/Lessons/11-03/yalefaces/subject10.normal.gif similarity index 100% rename from 11-03/yalefaces/subject10.normal.gif rename to Lessons/11-03/yalefaces/subject10.normal.gif diff --git a/11-03/yalefaces/subject10.rightlight.gif b/Lessons/11-03/yalefaces/subject10.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject10.rightlight.gif rename to Lessons/11-03/yalefaces/subject10.rightlight.gif diff --git a/11-03/yalefaces/subject10.sad.gif b/Lessons/11-03/yalefaces/subject10.sad.gif similarity index 100% rename from 11-03/yalefaces/subject10.sad.gif rename to Lessons/11-03/yalefaces/subject10.sad.gif diff --git a/11-03/yalefaces/subject10.sleepy.gif b/Lessons/11-03/yalefaces/subject10.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject10.sleepy.gif rename to Lessons/11-03/yalefaces/subject10.sleepy.gif diff --git a/11-03/yalefaces/subject10.surprised.gif b/Lessons/11-03/yalefaces/subject10.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject10.surprised.gif rename to Lessons/11-03/yalefaces/subject10.surprised.gif diff --git a/11-03/yalefaces/subject10.wink.gif b/Lessons/11-03/yalefaces/subject10.wink.gif similarity index 100% rename from 11-03/yalefaces/subject10.wink.gif rename to Lessons/11-03/yalefaces/subject10.wink.gif diff --git a/11-03/yalefaces/subject11.centerlight.gif b/Lessons/11-03/yalefaces/subject11.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject11.centerlight.gif rename to Lessons/11-03/yalefaces/subject11.centerlight.gif diff --git a/11-03/yalefaces/subject11.glasses.gif b/Lessons/11-03/yalefaces/subject11.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject11.glasses.gif rename to Lessons/11-03/yalefaces/subject11.glasses.gif diff --git a/11-03/yalefaces/subject11.happy.gif b/Lessons/11-03/yalefaces/subject11.happy.gif similarity index 100% rename from 11-03/yalefaces/subject11.happy.gif rename to Lessons/11-03/yalefaces/subject11.happy.gif diff --git a/11-03/yalefaces/subject11.leftlight.gif b/Lessons/11-03/yalefaces/subject11.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject11.leftlight.gif rename to Lessons/11-03/yalefaces/subject11.leftlight.gif diff --git a/11-03/yalefaces/subject11.noglasses.gif b/Lessons/11-03/yalefaces/subject11.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject11.noglasses.gif rename to Lessons/11-03/yalefaces/subject11.noglasses.gif diff --git a/11-03/yalefaces/subject11.normal.gif b/Lessons/11-03/yalefaces/subject11.normal.gif similarity index 100% rename from 11-03/yalefaces/subject11.normal.gif rename to Lessons/11-03/yalefaces/subject11.normal.gif diff --git a/11-03/yalefaces/subject11.rightlight.gif b/Lessons/11-03/yalefaces/subject11.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject11.rightlight.gif rename to Lessons/11-03/yalefaces/subject11.rightlight.gif diff --git a/11-03/yalefaces/subject11.sad.gif b/Lessons/11-03/yalefaces/subject11.sad.gif similarity index 100% rename from 11-03/yalefaces/subject11.sad.gif rename to Lessons/11-03/yalefaces/subject11.sad.gif diff --git a/11-03/yalefaces/subject11.sleepy.gif b/Lessons/11-03/yalefaces/subject11.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject11.sleepy.gif rename to Lessons/11-03/yalefaces/subject11.sleepy.gif diff --git a/11-03/yalefaces/subject11.surprised.gif b/Lessons/11-03/yalefaces/subject11.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject11.surprised.gif rename to Lessons/11-03/yalefaces/subject11.surprised.gif diff --git a/11-03/yalefaces/subject11.wink.gif b/Lessons/11-03/yalefaces/subject11.wink.gif similarity index 100% rename from 11-03/yalefaces/subject11.wink.gif rename to Lessons/11-03/yalefaces/subject11.wink.gif diff --git a/11-03/yalefaces/subject12.centerlight.gif b/Lessons/11-03/yalefaces/subject12.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject12.centerlight.gif rename to Lessons/11-03/yalefaces/subject12.centerlight.gif diff --git a/11-03/yalefaces/subject12.glasses.gif b/Lessons/11-03/yalefaces/subject12.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject12.glasses.gif rename to Lessons/11-03/yalefaces/subject12.glasses.gif diff --git a/11-03/yalefaces/subject12.happy.gif b/Lessons/11-03/yalefaces/subject12.happy.gif similarity index 100% rename from 11-03/yalefaces/subject12.happy.gif rename to Lessons/11-03/yalefaces/subject12.happy.gif diff --git a/11-03/yalefaces/subject12.leftlight.gif b/Lessons/11-03/yalefaces/subject12.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject12.leftlight.gif rename to Lessons/11-03/yalefaces/subject12.leftlight.gif diff --git a/11-03/yalefaces/subject12.noglasses.gif b/Lessons/11-03/yalefaces/subject12.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject12.noglasses.gif rename to Lessons/11-03/yalefaces/subject12.noglasses.gif diff --git a/11-03/yalefaces/subject12.normal.gif b/Lessons/11-03/yalefaces/subject12.normal.gif similarity index 100% rename from 11-03/yalefaces/subject12.normal.gif rename to Lessons/11-03/yalefaces/subject12.normal.gif diff --git a/11-03/yalefaces/subject12.rightlight.gif b/Lessons/11-03/yalefaces/subject12.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject12.rightlight.gif rename to Lessons/11-03/yalefaces/subject12.rightlight.gif diff --git a/11-03/yalefaces/subject12.sad.gif b/Lessons/11-03/yalefaces/subject12.sad.gif similarity index 100% rename from 11-03/yalefaces/subject12.sad.gif rename to Lessons/11-03/yalefaces/subject12.sad.gif diff --git a/11-03/yalefaces/subject12.sleepy.gif b/Lessons/11-03/yalefaces/subject12.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject12.sleepy.gif rename to Lessons/11-03/yalefaces/subject12.sleepy.gif diff --git a/11-03/yalefaces/subject12.surprised.gif b/Lessons/11-03/yalefaces/subject12.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject12.surprised.gif rename to Lessons/11-03/yalefaces/subject12.surprised.gif diff --git a/11-03/yalefaces/subject12.wink.gif b/Lessons/11-03/yalefaces/subject12.wink.gif similarity index 100% rename from 11-03/yalefaces/subject12.wink.gif rename to Lessons/11-03/yalefaces/subject12.wink.gif diff --git a/11-03/yalefaces/subject13.centerlight.gif b/Lessons/11-03/yalefaces/subject13.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject13.centerlight.gif rename to Lessons/11-03/yalefaces/subject13.centerlight.gif diff --git a/11-03/yalefaces/subject13.glasses.gif b/Lessons/11-03/yalefaces/subject13.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject13.glasses.gif rename to Lessons/11-03/yalefaces/subject13.glasses.gif diff --git a/11-03/yalefaces/subject13.happy.gif b/Lessons/11-03/yalefaces/subject13.happy.gif similarity index 100% rename from 11-03/yalefaces/subject13.happy.gif rename to Lessons/11-03/yalefaces/subject13.happy.gif diff --git a/11-03/yalefaces/subject13.leftlight.gif b/Lessons/11-03/yalefaces/subject13.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject13.leftlight.gif rename to Lessons/11-03/yalefaces/subject13.leftlight.gif diff --git a/11-03/yalefaces/subject13.noglasses.gif b/Lessons/11-03/yalefaces/subject13.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject13.noglasses.gif rename to Lessons/11-03/yalefaces/subject13.noglasses.gif diff --git a/11-03/yalefaces/subject13.normal.gif b/Lessons/11-03/yalefaces/subject13.normal.gif similarity index 100% rename from 11-03/yalefaces/subject13.normal.gif rename to Lessons/11-03/yalefaces/subject13.normal.gif diff --git a/11-03/yalefaces/subject13.rightlight.gif b/Lessons/11-03/yalefaces/subject13.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject13.rightlight.gif rename to Lessons/11-03/yalefaces/subject13.rightlight.gif diff --git a/11-03/yalefaces/subject13.sad.gif b/Lessons/11-03/yalefaces/subject13.sad.gif similarity index 100% rename from 11-03/yalefaces/subject13.sad.gif rename to Lessons/11-03/yalefaces/subject13.sad.gif diff --git a/11-03/yalefaces/subject13.sleepy.gif b/Lessons/11-03/yalefaces/subject13.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject13.sleepy.gif rename to Lessons/11-03/yalefaces/subject13.sleepy.gif diff --git a/11-03/yalefaces/subject13.surprised.gif b/Lessons/11-03/yalefaces/subject13.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject13.surprised.gif rename to Lessons/11-03/yalefaces/subject13.surprised.gif diff --git a/11-03/yalefaces/subject13.wink.gif b/Lessons/11-03/yalefaces/subject13.wink.gif similarity index 100% rename from 11-03/yalefaces/subject13.wink.gif rename to Lessons/11-03/yalefaces/subject13.wink.gif diff --git a/11-03/yalefaces/subject14.centerlight.gif b/Lessons/11-03/yalefaces/subject14.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject14.centerlight.gif rename to Lessons/11-03/yalefaces/subject14.centerlight.gif diff --git a/11-03/yalefaces/subject14.glasses.gif b/Lessons/11-03/yalefaces/subject14.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject14.glasses.gif rename to Lessons/11-03/yalefaces/subject14.glasses.gif diff --git a/11-03/yalefaces/subject14.happy.gif b/Lessons/11-03/yalefaces/subject14.happy.gif similarity index 100% rename from 11-03/yalefaces/subject14.happy.gif rename to Lessons/11-03/yalefaces/subject14.happy.gif diff --git a/11-03/yalefaces/subject14.leftlight.gif b/Lessons/11-03/yalefaces/subject14.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject14.leftlight.gif rename to Lessons/11-03/yalefaces/subject14.leftlight.gif diff --git a/11-03/yalefaces/subject14.noglasses.gif b/Lessons/11-03/yalefaces/subject14.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject14.noglasses.gif rename to Lessons/11-03/yalefaces/subject14.noglasses.gif diff --git a/11-03/yalefaces/subject14.normal.gif b/Lessons/11-03/yalefaces/subject14.normal.gif similarity index 100% rename from 11-03/yalefaces/subject14.normal.gif rename to Lessons/11-03/yalefaces/subject14.normal.gif diff --git a/11-03/yalefaces/subject14.rightlight.gif b/Lessons/11-03/yalefaces/subject14.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject14.rightlight.gif rename to Lessons/11-03/yalefaces/subject14.rightlight.gif diff --git a/11-03/yalefaces/subject14.sad.gif b/Lessons/11-03/yalefaces/subject14.sad.gif similarity index 100% rename from 11-03/yalefaces/subject14.sad.gif rename to Lessons/11-03/yalefaces/subject14.sad.gif diff --git a/11-03/yalefaces/subject14.sleepy.gif b/Lessons/11-03/yalefaces/subject14.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject14.sleepy.gif rename to Lessons/11-03/yalefaces/subject14.sleepy.gif diff --git a/11-03/yalefaces/subject14.surprised.gif b/Lessons/11-03/yalefaces/subject14.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject14.surprised.gif rename to Lessons/11-03/yalefaces/subject14.surprised.gif diff --git a/11-03/yalefaces/subject14.wink.gif b/Lessons/11-03/yalefaces/subject14.wink.gif similarity index 100% rename from 11-03/yalefaces/subject14.wink.gif rename to Lessons/11-03/yalefaces/subject14.wink.gif diff --git a/11-03/yalefaces/subject15.centerlight.gif b/Lessons/11-03/yalefaces/subject15.centerlight.gif similarity index 100% rename from 11-03/yalefaces/subject15.centerlight.gif rename to Lessons/11-03/yalefaces/subject15.centerlight.gif diff --git a/11-03/yalefaces/subject15.glasses.gif b/Lessons/11-03/yalefaces/subject15.glasses.gif similarity index 100% rename from 11-03/yalefaces/subject15.glasses.gif rename to Lessons/11-03/yalefaces/subject15.glasses.gif diff --git a/11-03/yalefaces/subject15.happy.gif b/Lessons/11-03/yalefaces/subject15.happy.gif similarity index 100% rename from 11-03/yalefaces/subject15.happy.gif rename to Lessons/11-03/yalefaces/subject15.happy.gif diff --git a/11-03/yalefaces/subject15.leftlight.gif b/Lessons/11-03/yalefaces/subject15.leftlight.gif similarity index 100% rename from 11-03/yalefaces/subject15.leftlight.gif rename to Lessons/11-03/yalefaces/subject15.leftlight.gif diff --git a/11-03/yalefaces/subject15.noglasses.gif b/Lessons/11-03/yalefaces/subject15.noglasses.gif similarity index 100% rename from 11-03/yalefaces/subject15.noglasses.gif rename to Lessons/11-03/yalefaces/subject15.noglasses.gif diff --git a/11-03/yalefaces/subject15.normal.gif b/Lessons/11-03/yalefaces/subject15.normal.gif similarity index 100% rename from 11-03/yalefaces/subject15.normal.gif rename to Lessons/11-03/yalefaces/subject15.normal.gif diff --git a/11-03/yalefaces/subject15.rightlight.gif b/Lessons/11-03/yalefaces/subject15.rightlight.gif similarity index 100% rename from 11-03/yalefaces/subject15.rightlight.gif rename to Lessons/11-03/yalefaces/subject15.rightlight.gif diff --git a/11-03/yalefaces/subject15.sad.gif b/Lessons/11-03/yalefaces/subject15.sad.gif similarity index 100% rename from 11-03/yalefaces/subject15.sad.gif rename to Lessons/11-03/yalefaces/subject15.sad.gif diff --git a/11-03/yalefaces/subject15.sleepy.gif b/Lessons/11-03/yalefaces/subject15.sleepy.gif similarity index 100% rename from 11-03/yalefaces/subject15.sleepy.gif rename to Lessons/11-03/yalefaces/subject15.sleepy.gif diff --git a/11-03/yalefaces/subject15.surprised.gif b/Lessons/11-03/yalefaces/subject15.surprised.gif similarity index 100% rename from 11-03/yalefaces/subject15.surprised.gif rename to Lessons/11-03/yalefaces/subject15.surprised.gif diff --git a/11-03/yalefaces/subject15.wink.gif b/Lessons/11-03/yalefaces/subject15.wink.gif similarity index 100% rename from 11-03/yalefaces/subject15.wink.gif rename to Lessons/11-03/yalefaces/subject15.wink.gif diff --git a/11-08/eigenfaces_scatter.jl b/Lessons/11-08/eigenfaces_scatter.jl similarity index 100% rename from 11-08/eigenfaces_scatter.jl rename to Lessons/11-08/eigenfaces_scatter.jl diff --git a/11-08/interactiverec.jl b/Lessons/11-08/interactiverec.jl similarity index 100% rename from 11-08/interactiverec.jl rename to Lessons/11-08/interactiverec.jl diff --git a/11-08/lesson.ipynb b/Lessons/11-08/lesson.ipynb similarity index 100% rename from 11-08/lesson.ipynb rename to Lessons/11-08/lesson.ipynb diff --git a/11-09/NWTN.jl b/Lessons/11-09/NWTN.jl similarity index 100% rename from 11-09/NWTN.jl rename to Lessons/11-09/NWTN.jl diff --git a/11-09/NWTN.m b/Lessons/11-09/NWTN.m similarity index 100% rename from 11-09/NWTN.m rename to Lessons/11-09/NWTN.m diff --git a/11-09/SDG.jl b/Lessons/11-09/SDG.jl similarity index 100% rename from 11-09/SDG.jl rename to Lessons/11-09/SDG.jl diff --git a/11-09/SDG.m b/Lessons/11-09/SDG.m similarity index 100% rename from 11-09/SDG.m rename to Lessons/11-09/SDG.m diff --git a/11-09/TestFunctions Matlab/TestFunctions.m b/Lessons/11-09/TestFunctions Matlab/TestFunctions.m similarity index 100% rename from 11-09/TestFunctions Matlab/TestFunctions.m rename to Lessons/11-09/TestFunctions Matlab/TestFunctions.m diff --git a/11-09/TestFunctions Matlab/roughNN.m b/Lessons/11-09/TestFunctions Matlab/roughNN.m similarity index 100% rename from 11-09/TestFunctions Matlab/roughNN.m rename to Lessons/11-09/TestFunctions Matlab/roughNN.m diff --git a/11-09/TestFunctions Matlab/testNN.m b/Lessons/11-09/TestFunctions Matlab/testNN.m similarity index 100% rename from 11-09/TestFunctions Matlab/testNN.m rename to Lessons/11-09/TestFunctions Matlab/testNN.m diff --git a/11-09/TestFunctions Matlab/testNN_ADiGatorGrd.m b/Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorGrd.m similarity index 100% rename from 11-09/TestFunctions Matlab/testNN_ADiGatorGrd.m rename to Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorGrd.m diff --git a/11-09/TestFunctions Matlab/testNN_ADiGatorGrd.mat b/Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorGrd.mat similarity index 100% rename from 11-09/TestFunctions Matlab/testNN_ADiGatorGrd.mat rename to Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorGrd.mat diff --git a/11-09/TestFunctions Matlab/testNN_ADiGatorHes.m b/Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorHes.m similarity index 100% rename from 11-09/TestFunctions Matlab/testNN_ADiGatorHes.m rename to Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorHes.m diff --git a/11-09/TestFunctions Matlab/testNN_ADiGatorHes.mat b/Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorHes.mat similarity index 100% rename from 11-09/TestFunctions Matlab/testNN_ADiGatorHes.mat rename to Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorHes.mat diff --git a/11-09/TestFunctions Matlab/testNN_ADiGatorJac.m b/Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorJac.m similarity index 100% rename from 11-09/TestFunctions Matlab/testNN_ADiGatorJac.m rename to Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorJac.m diff --git a/11-09/TestFunctions Matlab/testNN_ADiGatorJac.mat b/Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorJac.mat similarity index 100% rename from 11-09/TestFunctions Matlab/testNN_ADiGatorJac.mat rename to Lessons/11-09/TestFunctions Matlab/testNN_ADiGatorJac.mat diff --git a/11-09/TestFunctions Matlab/testNN_Grd.m b/Lessons/11-09/TestFunctions Matlab/testNN_Grd.m similarity index 100% rename from 11-09/TestFunctions Matlab/testNN_Grd.m rename to Lessons/11-09/TestFunctions Matlab/testNN_Grd.m diff --git a/11-09/TestFunctions Matlab/testNN_Hes.m b/Lessons/11-09/TestFunctions Matlab/testNN_Hes.m similarity index 100% rename from 11-09/TestFunctions Matlab/testNN_Hes.m rename to Lessons/11-09/TestFunctions Matlab/testNN_Hes.m diff --git a/11-09/TestFunctions Matlab/testNN_Jac.m b/Lessons/11-09/TestFunctions Matlab/testNN_Jac.m similarity index 100% rename from 11-09/TestFunctions Matlab/testNN_Jac.m rename to Lessons/11-09/TestFunctions Matlab/testNN_Jac.m diff --git a/11-09/TestFunctions/TestFunctions.jl b/Lessons/11-09/TestFunctions/TestFunctions.jl similarity index 100% rename from 11-09/TestFunctions/TestFunctions.jl rename to Lessons/11-09/TestFunctions/TestFunctions.jl diff --git a/11-09/TestFunctions/roughNN.jl b/Lessons/11-09/TestFunctions/roughNN.jl similarity index 100% rename from 11-09/TestFunctions/roughNN.jl rename to Lessons/11-09/TestFunctions/roughNN.jl diff --git a/11-09/TestFunctions/testNN.jl b/Lessons/11-09/TestFunctions/testNN.jl similarity index 100% rename from 11-09/TestFunctions/testNN.jl rename to Lessons/11-09/TestFunctions/testNN.jl diff --git a/11-09/TestFunctions/testNN_Hes.jl b/Lessons/11-09/TestFunctions/testNN_Hes.jl similarity index 100% rename from 11-09/TestFunctions/testNN_Hes.jl rename to Lessons/11-09/TestFunctions/testNN_Hes.jl diff --git a/11-09/TestFunctions/testNN_Jac.jl b/Lessons/11-09/TestFunctions/testNN_Jac.jl similarity index 100% rename from 11-09/TestFunctions/testNN_Jac.jl rename to Lessons/11-09/TestFunctions/testNN_Jac.jl diff --git a/11-09/lesson.ipynb b/Lessons/11-09/lesson.ipynb similarity index 100% rename from 11-09/lesson.ipynb rename to Lessons/11-09/lesson.ipynb diff --git a/11-10/lesson.ipynb b/Lessons/11-10/lesson.ipynb similarity index 100% rename from 11-10/lesson.ipynb rename to Lessons/11-10/lesson.ipynb diff --git a/11-15/lesson.ipynb b/Lessons/11-15/lesson.ipynb similarity index 100% rename from 11-15/lesson.ipynb rename to Lessons/11-15/lesson.ipynb diff --git a/11-17/lesson.ipynb b/Lessons/11-17/lesson.ipynb similarity index 100% rename from 11-17/lesson.ipynb rename to Lessons/11-17/lesson.ipynb diff --git a/11-19/lesson.ipynb b/Lessons/11-19/lesson.ipynb similarity index 100% rename from 11-19/lesson.ipynb rename to Lessons/11-19/lesson.ipynb diff --git a/11-22/NCG.jl b/Lessons/11-22/NCG.jl similarity index 100% rename from 11-22/NCG.jl rename to Lessons/11-22/NCG.jl diff --git a/11-22/lesson.ipynb b/Lessons/11-22/lesson.ipynb similarity index 100% rename from 11-22/lesson.ipynb rename to Lessons/11-22/lesson.ipynb diff --git a/11-23/HBG.jl b/Lessons/11-23/HBG.jl similarity index 100% rename from 11-23/HBG.jl rename to Lessons/11-23/HBG.jl diff --git a/11-23/NCG.jl b/Lessons/11-23/NCG.jl similarity index 100% rename from 11-23/NCG.jl rename to Lessons/11-23/NCG.jl diff --git a/11-23/SGM.jl b/Lessons/11-23/SGM.jl similarity index 100% rename from 11-23/SGM.jl rename to Lessons/11-23/SGM.jl diff --git a/11-23/lesson.ipynb b/Lessons/11-23/lesson.ipynb similarity index 100% rename from 11-23/lesson.ipynb rename to Lessons/11-23/lesson.ipynb diff --git a/README.md b/README.md index 899979a..f938eaf 100644 --- a/README.md +++ b/README.md @@ -2,18 +2,18 @@ Repository for the course Computational Mathematics for Learning and Data Analysis. -# Installation +## Installation These steps are for MacOS, with [```brew```](brew.sh) and ```python3``` already installed. -## Installing Julia +### Installing Julia ```sh brew install julia ``` -## Installing JupyterLab and IJulia kernel +### Installing JupyterLab and IJulia kernel -### IJulia kernel +#### IJulia kernel Inside the julia REPL: ```julia @@ -23,7 +23,7 @@ Pkg.add("IJulia") A simple notebook experience can be obtained by simply running ```noteboot()```, which will install in a separate [```conda```](https://docs.conda.io/en/latest/) environment the needed python packages. -### JupyterLab +#### JupyterLab A separate environment with [```virtualenv```](https://pypi.org/project/virtualenv/) or [```virtualenvwrapper```](https://pypi.org/project/virtualenvwrapper/) is recommended. @@ -31,6 +31,39 @@ A separate environment with [```virtualenv```](https://pypi.org/project/virtuale pip install jupyterlab ``` -# Running +## Running To run simply start the notebook with ```jupyter lab``` , the julia kernel should be automatically detected. + + +# Project 19 (Non-ML) + +## Project + +In the folder `Project` there is our implementation for the project. + +Project description: + +> (P) is the linear least squares problem + +> $$\min_{w} \lVert \hat{X}w-\hat{y} \rVert$$ + +> where + +> $$\hat{X}= \begin{bmatrix} X^T \newline \lambda I_m \end{bmatrix},\ \ \hat{y} = \begin{bmatrix} y \newline 0 \end{bmatrix},$$ + +> with $X$ the (tall thin) matrix from the ML-cup dataset by prof. Micheli, $\lambda > 0$ and $y$ is a random vector. + +> - (A1) is an algorithm of the class of **limited-memory quasi-Newton methods**. +> - (A2) is **thin QR factorization with Householder reflectors**, in the variant where one does not form the matrix $Q$, but stores the Householder vectors $u_k$ and uses them to perform (implicitly) products with $Q$ and $Q^T$. + +## Report + +In the folder `Report` there is the latex project for the report of the project. + +Run to compile: +``` +pdflatex -file-line-error -shell-escape --synctex=1 -interaction=nonstopmode main.tex +biber main +pdflatex -file-line-error -shell-escape --synctex=1 -interaction=nonstopmode main.tex +``` diff --git a/Report/(1) - introduction/introduction.aux b/Report/(1) - introduction/introduction.aux new file mode 100644 index 0000000..a206ac4 --- /dev/null +++ b/Report/(1) - introduction/introduction.aux @@ -0,0 +1,168 @@ +\relax +\providecommand\hyper@newdestlabel[2]{} +\@writefile{toc}{\contentsline {chapter}{\numberline {1}Introduction}{1}{chapter.1}\protected@file@percent } +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loa}{\addvspace {10\p@ }} +\newlabel{ch: introduction}{{1}{1}{Introduction}{chapter.1}{}} +\@setckpt{(1) - introduction/introduction}{ +\setcounter{page}{2} +\setcounter{equation}{0} +\setcounter{enumi}{0} +\setcounter{enumii}{0} +\setcounter{enumiii}{0} +\setcounter{enumiv}{0} +\setcounter{footnote}{0} +\setcounter{mpfootnote}{0} +\setcounter{part}{0} +\setcounter{chapter}{1} +\setcounter{section}{0} +\setcounter{subsection}{0} +\setcounter{subsubsection}{0} +\setcounter{paragraph}{0} +\setcounter{subparagraph}{0} +\setcounter{figure}{0} +\setcounter{table}{0} +\setcounter{tabx@nest}{0} +\setcounter{listtotal}{0} +\setcounter{listcount}{0} +\setcounter{liststart}{0} +\setcounter{liststop}{0} +\setcounter{citecount}{0} +\setcounter{citetotal}{0} +\setcounter{multicitecount}{0} +\setcounter{multicitetotal}{0} +\setcounter{instcount}{0} +\setcounter{maxnames}{3} +\setcounter{minnames}{3} +\setcounter{maxitems}{3} +\setcounter{minitems}{1} +\setcounter{citecounter}{0} +\setcounter{maxcitecounter}{0} +\setcounter{savedcitecounter}{0} +\setcounter{uniquelist}{0} +\setcounter{uniquename}{0} +\setcounter{refsection}{0} +\setcounter{refsegment}{0} +\setcounter{maxextratitle}{0} +\setcounter{maxextratitleyear}{0} +\setcounter{maxextraname}{0} +\setcounter{maxextradate}{0} +\setcounter{maxextraalpha}{0} +\setcounter{abbrvpenalty}{50} +\setcounter{highnamepenalty}{50} +\setcounter{lownamepenalty}{25} +\setcounter{maxparens}{3} +\setcounter{parenlevel}{0} +\setcounter{blx@maxsection}{0} +\setcounter{mincomprange}{10} +\setcounter{maxcomprange}{100000} +\setcounter{mincompwidth}{1} +\setcounter{afterword}{0} +\setcounter{savedafterword}{0} +\setcounter{annotator}{0} +\setcounter{savedannotator}{0} +\setcounter{author}{0} +\setcounter{savedauthor}{0} +\setcounter{bookauthor}{0} +\setcounter{savedbookauthor}{0} +\setcounter{commentator}{0} +\setcounter{savedcommentator}{0} +\setcounter{editor}{0} +\setcounter{savededitor}{0} +\setcounter{editora}{0} +\setcounter{savededitora}{0} +\setcounter{editorb}{0} +\setcounter{savededitorb}{0} +\setcounter{editorc}{0} +\setcounter{savededitorc}{0} +\setcounter{foreword}{0} +\setcounter{savedforeword}{0} +\setcounter{holder}{0} +\setcounter{savedholder}{0} +\setcounter{introduction}{0} +\setcounter{savedintroduction}{0} +\setcounter{namea}{0} +\setcounter{savednamea}{0} +\setcounter{nameb}{0} +\setcounter{savednameb}{0} +\setcounter{namec}{0} +\setcounter{savednamec}{0} +\setcounter{translator}{0} +\setcounter{savedtranslator}{0} +\setcounter{shortauthor}{0} +\setcounter{savedshortauthor}{0} +\setcounter{shorteditor}{0} +\setcounter{savedshorteditor}{0} +\setcounter{labelname}{0} +\setcounter{savedlabelname}{0} +\setcounter{institution}{0} +\setcounter{savedinstitution}{0} +\setcounter{lista}{0} +\setcounter{savedlista}{0} +\setcounter{listb}{0} +\setcounter{savedlistb}{0} +\setcounter{listc}{0} +\setcounter{savedlistc}{0} +\setcounter{listd}{0} +\setcounter{savedlistd}{0} +\setcounter{liste}{0} +\setcounter{savedliste}{0} +\setcounter{listf}{0} +\setcounter{savedlistf}{0} +\setcounter{location}{0} +\setcounter{savedlocation}{0} +\setcounter{organization}{0} +\setcounter{savedorganization}{0} +\setcounter{origlocation}{0} +\setcounter{savedoriglocation}{0} +\setcounter{origpublisher}{0} +\setcounter{savedorigpublisher}{0} +\setcounter{publisher}{0} +\setcounter{savedpublisher}{0} +\setcounter{language}{0} +\setcounter{savedlanguage}{0} +\setcounter{origlanguage}{0} +\setcounter{savedoriglanguage}{0} +\setcounter{pageref}{0} +\setcounter{savedpageref}{0} +\setcounter{textcitecount}{0} +\setcounter{textcitetotal}{0} +\setcounter{textcitemaxnames}{0} +\setcounter{biburlbigbreakpenalty}{100} +\setcounter{biburlbreakpenalty}{200} +\setcounter{biburlnumpenalty}{0} +\setcounter{biburlucpenalty}{0} +\setcounter{biburllcpenalty}{0} +\setcounter{smartand}{1} +\setcounter{bbx:relatedcount}{0} +\setcounter{bbx:relatedtotal}{0} +\setcounter{cbx@tempcnta}{0} +\setcounter{cbx@tempcntb}{0} +\setcounter{cbx@tempcntc}{0} +\setcounter{cbx@tempcntd}{0} +\setcounter{float@type}{4} +\setcounter{parentequation}{0} +\setcounter{ALG@line}{0} +\setcounter{ALG@rem}{0} +\setcounter{ALG@nested}{0} +\setcounter{ALG@Lnr}{2} +\setcounter{ALG@blocknr}{10} +\setcounter{ALG@storecount}{0} +\setcounter{ALG@tmpcounter}{0} +\setcounter{thmt@dummyctr}{0} +\setcounter{nlinenum}{0} +\setcounter{caption@flags}{0} +\setcounter{continuedfloat}{0} +\setcounter{subfigure}{0} +\setcounter{subtable}{0} +\setcounter{section@level}{0} +\setcounter{Item}{0} +\setcounter{Hfootnote}{0} +\setcounter{bookmark@seq@number}{0} +\setcounter{AlgoLine}{0} +\setcounter{algocfline}{0} +\setcounter{algocfproc}{0} +\setcounter{algocf}{0} +\setcounter{mlemma}{0} +} diff --git a/Report/(1) - introduction/introduction.tex b/Report/(1) - introduction/introduction.tex new file mode 100644 index 0000000..5acee76 --- /dev/null +++ b/Report/(1) - introduction/introduction.tex @@ -0,0 +1,33 @@ +\chapter{Introduction}\label{ch: introduction} + +(P) is the linear least squares problem +\[\min_{w}\ \left\lVert \hat{X}w-\hat{y} \right\rVert\] +where +\[ + \hat{X} = + \begin{bmatrix} + X^T \\ + \lambda I_m + \end{bmatrix}, + \ \ + \hat{y} = + \begin{bmatrix} + y \\ + 0 + \end{bmatrix}, +\] +with $X$ the (tall thin) matrix from the ML-cup dataset by prof. Micheli, $\lambda > 0$ and $y$ is a random vector. +\begin{itemize} +\item[--] (A1) is an algorithm of the class of limited-memory quasi-Newton methods. +\item[--] (A2) is a cothin QR factorization with Householder reflectors, in the variant where one does not form the matrix $Q$, but stores the Householder vectors $u_k$ and uses them to perform (implicitly) products with $Q$ and $Q^T$. +\end{itemize} +No off-the-shelf solvers allowed. In particular you must implement yourself the thin QR factorization, and the computational cost of your implementation should be at most quadratic in $m$. + +\subsection*{Outline} +This report is organized as follows: +\begin{description} +\item[\autoref{ch: problem definition},] in which the problem is reformulated under the mathematical aspect; +\item[\autoref{ch: algorithms},] where we will include the implemented algorithms, with the analysis of convergence and complexity; +\item[\autoref{ch: experiments},] to evaluate and compare (A1) with (A2) for this task and provide different tests in order to examine deeper the algorithms; +\item[\autoref{ch: conclusion},] in which conclusions are drawn, offering a critical analysis of the results obtained. +\end{description} diff --git a/Report/(2) - problem definition/images/conditioning.png b/Report/(2) - problem definition/images/conditioning.png new file mode 100644 index 0000000..672c39d Binary files /dev/null and b/Report/(2) - problem definition/images/conditioning.png differ diff --git a/Report/(2) - problem definition/problem definition.aux b/Report/(2) - problem definition/problem definition.aux new file mode 100644 index 0000000..5fa8095 --- /dev/null +++ b/Report/(2) - problem definition/problem definition.aux @@ -0,0 +1,177 @@ +\relax +\providecommand\hyper@newdestlabel[2]{} +\@writefile{toc}{\contentsline {chapter}{\numberline {2}Problem Definition}{2}{chapter.2}\protected@file@percent } +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loa}{\addvspace {10\p@ }} +\newlabel{ch: problem definition}{{2}{2}{Problem Definition}{chapter.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {2.1}QR}{2}{section.2.1}\protected@file@percent } +\@writefile{toc}{\contentsline {section}{\numberline {2.2}L-BFGS}{3}{section.2.2}\protected@file@percent } +\newlabel{ch: L-BFGS}{{2.2}{3}{L-BFGS}{section.2.2}{}} +\newlabel{definitions: hessian tomography}{{2.2}{4}{L-BFGS}{equation.2.2.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {2.3}Conditioning}{4}{section.2.3}\protected@file@percent } +\newlabel{subsec:conditioning}{{2.3}{4}{Conditioning}{section.2.3}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {2.1}{\ignorespaces $\kappa (\hat {X})$ \textit {for different values of} $\lambda $}}{5}{figure.caption.2}\protected@file@percent } +\providecommand*\caption@xref[2]{\@setref\relax\@undefined{#1}} +\newlabel{fig:condition}{{2.1}{5}{$\kappa (\hat {X})$ \textit {for different values of} $\lambda $}{figure.caption.2}{}} +\@setckpt{(2) - problem definition/problem definition}{ +\setcounter{page}{6} +\setcounter{equation}{2} +\setcounter{enumi}{0} +\setcounter{enumii}{0} +\setcounter{enumiii}{0} +\setcounter{enumiv}{0} +\setcounter{footnote}{0} +\setcounter{mpfootnote}{0} +\setcounter{part}{0} +\setcounter{chapter}{2} +\setcounter{section}{3} +\setcounter{subsection}{0} +\setcounter{subsubsection}{0} +\setcounter{paragraph}{0} +\setcounter{subparagraph}{0} +\setcounter{figure}{1} +\setcounter{table}{0} +\setcounter{tabx@nest}{0} +\setcounter{listtotal}{0} +\setcounter{listcount}{0} +\setcounter{liststart}{0} +\setcounter{liststop}{0} +\setcounter{citecount}{0} +\setcounter{citetotal}{0} +\setcounter{multicitecount}{0} +\setcounter{multicitetotal}{0} +\setcounter{instcount}{0} +\setcounter{maxnames}{3} +\setcounter{minnames}{3} +\setcounter{maxitems}{3} +\setcounter{minitems}{1} +\setcounter{citecounter}{0} +\setcounter{maxcitecounter}{0} +\setcounter{savedcitecounter}{0} +\setcounter{uniquelist}{0} +\setcounter{uniquename}{0} +\setcounter{refsection}{0} +\setcounter{refsegment}{0} +\setcounter{maxextratitle}{0} +\setcounter{maxextratitleyear}{0} +\setcounter{maxextraname}{0} +\setcounter{maxextradate}{0} +\setcounter{maxextraalpha}{0} +\setcounter{abbrvpenalty}{50} +\setcounter{highnamepenalty}{50} +\setcounter{lownamepenalty}{25} +\setcounter{maxparens}{3} +\setcounter{parenlevel}{0} +\setcounter{blx@maxsection}{0} +\setcounter{mincomprange}{10} +\setcounter{maxcomprange}{100000} +\setcounter{mincompwidth}{1} +\setcounter{afterword}{0} +\setcounter{savedafterword}{0} +\setcounter{annotator}{0} +\setcounter{savedannotator}{0} +\setcounter{author}{0} +\setcounter{savedauthor}{0} +\setcounter{bookauthor}{0} +\setcounter{savedbookauthor}{0} +\setcounter{commentator}{0} +\setcounter{savedcommentator}{0} +\setcounter{editor}{0} +\setcounter{savededitor}{0} +\setcounter{editora}{0} +\setcounter{savededitora}{0} +\setcounter{editorb}{0} +\setcounter{savededitorb}{0} +\setcounter{editorc}{0} +\setcounter{savededitorc}{0} +\setcounter{foreword}{0} +\setcounter{savedforeword}{0} +\setcounter{holder}{0} +\setcounter{savedholder}{0} +\setcounter{introduction}{0} +\setcounter{savedintroduction}{0} +\setcounter{namea}{0} +\setcounter{savednamea}{0} +\setcounter{nameb}{0} +\setcounter{savednameb}{0} +\setcounter{namec}{0} +\setcounter{savednamec}{0} +\setcounter{translator}{0} +\setcounter{savedtranslator}{0} +\setcounter{shortauthor}{0} +\setcounter{savedshortauthor}{0} +\setcounter{shorteditor}{0} +\setcounter{savedshorteditor}{0} +\setcounter{labelname}{0} +\setcounter{savedlabelname}{0} +\setcounter{institution}{0} +\setcounter{savedinstitution}{0} +\setcounter{lista}{0} +\setcounter{savedlista}{0} +\setcounter{listb}{0} +\setcounter{savedlistb}{0} +\setcounter{listc}{0} +\setcounter{savedlistc}{0} +\setcounter{listd}{0} +\setcounter{savedlistd}{0} +\setcounter{liste}{0} +\setcounter{savedliste}{0} +\setcounter{listf}{0} +\setcounter{savedlistf}{0} +\setcounter{location}{0} +\setcounter{savedlocation}{0} +\setcounter{organization}{0} +\setcounter{savedorganization}{0} +\setcounter{origlocation}{0} +\setcounter{savedoriglocation}{0} +\setcounter{origpublisher}{0} +\setcounter{savedorigpublisher}{0} +\setcounter{publisher}{0} +\setcounter{savedpublisher}{0} +\setcounter{language}{0} +\setcounter{savedlanguage}{0} +\setcounter{origlanguage}{0} +\setcounter{savedoriglanguage}{0} +\setcounter{pageref}{0} +\setcounter{savedpageref}{0} +\setcounter{textcitecount}{0} +\setcounter{textcitetotal}{0} +\setcounter{textcitemaxnames}{0} +\setcounter{biburlbigbreakpenalty}{100} +\setcounter{biburlbreakpenalty}{200} +\setcounter{biburlnumpenalty}{0} +\setcounter{biburlucpenalty}{0} +\setcounter{biburllcpenalty}{0} +\setcounter{smartand}{1} +\setcounter{bbx:relatedcount}{0} +\setcounter{bbx:relatedtotal}{0} +\setcounter{cbx@tempcnta}{0} +\setcounter{cbx@tempcntb}{0} +\setcounter{cbx@tempcntc}{0} +\setcounter{cbx@tempcntd}{0} +\setcounter{float@type}{4} +\setcounter{parentequation}{0} +\setcounter{ALG@line}{0} +\setcounter{ALG@rem}{0} +\setcounter{ALG@nested}{0} +\setcounter{ALG@Lnr}{2} +\setcounter{ALG@blocknr}{10} +\setcounter{ALG@storecount}{0} +\setcounter{ALG@tmpcounter}{0} +\setcounter{thmt@dummyctr}{0} +\setcounter{nlinenum}{0} +\setcounter{caption@flags}{2} +\setcounter{continuedfloat}{0} +\setcounter{subfigure}{0} +\setcounter{subtable}{0} +\setcounter{section@level}{0} +\setcounter{Item}{0} +\setcounter{Hfootnote}{0} +\setcounter{bookmark@seq@number}{0} +\setcounter{AlgoLine}{0} +\setcounter{algocfline}{0} +\setcounter{algocfproc}{0} +\setcounter{algocf}{0} +\setcounter{mlemma}{0} +} diff --git a/Report/(2) - problem definition/problem definition.tex b/Report/(2) - problem definition/problem definition.tex new file mode 100644 index 0000000..c2b7b7d --- /dev/null +++ b/Report/(2) - problem definition/problem definition.tex @@ -0,0 +1,147 @@ +\chapter{Problem Definition}\label{ch: problem definition} +Henceforth, we denote the norm-2 $\norm{ - }_2$ with the generic norm symbol $\norm{ - }$.\newline +Given $\hat{X} \in \mathbb{R}^{(m + n) \times m},\ \hat{y} \in \mathbb{R}^{m + n},\ $ we want to find +\[\min_{w}\ \norm{\hat{X}w-\hat{y}}\] + +\section{QR} +By performing a QR factorization on $\hat{X}$ we can reformulate the problem as follows: +\[ + \min_{w}\ \norm{\hat{X}w - \hat{y}} = \min_{w}\ \norm{\vphantom{\hat{X}}QRw - \hat{y}} +\] +with $Q \in \mathbb{R}^{(m + n) \times (m + n)}$ being an orthogonal matrix and $R \in \mathbb{R}^{(m + n) \times m}$ being an upper triangular matrix. Knowing that $R_{ij} = 0,\ \ \forall i > j,\ i = 1, \ldots, m + n,\ j = 1, \ldots, m,\ $ we can write +\begin{equation*} + \begin{aligned} + &R = + \begin{bmatrix} + R_0 \\ + 0 + \end{bmatrix},\ + &R_0 \in \mathbb{R}^{m \times m} \\ + &Q = + \begin{bmatrix} + Q_0\ Q_c + \end{bmatrix},\ + &Q_0 \in \mathbb{R}^{(m+n) \times m},\ &Q_c \in \mathbb{R}^{(m+n) \times n} + \end{aligned} +\end{equation*} +Since orthogonal matrices preserve norm-2, we have: +\begin{equation*} + \begin{aligned} + &\min_{w}\ \norm{QRw - \hat{y}} = \min_{w}\ \norm{Q^T(QRw - \hat{y})} = \\ + &\min_{w}\ \norm{Q^{T}QRw - Q^T\hat{y}} = \\ + &\min_{w}\ \norm{Rw - Q^T\hat{y}} = \\ + &\min_{w}\ \norm{ + \begin{bmatrix} + R_0 \\ + 0 + \end{bmatrix} + w - + \begin{bmatrix} + Q^T_0 \\ + Q^T_c + \end{bmatrix}\hat{y}} = \\ + & \min_{w}\ \norm{ + \begin{bmatrix} + R_0w - Q^T_0\hat{y} \\ + - Q^T_c\hat{y} + \end{bmatrix}} + \end{aligned} +\end{equation*} +The entries of the second block $- Q^T_c\hat{y}$ do not depend on $w$, meaning that they will appear in the norm independently from $w$. Thus, we can simplify the problem and solve the triangular system +\begin{equation*} + R_0w - Q^T_0\hat{y} = 0 \iff R_0w = Q^T_0\hat{y} +\end{equation*} +provided that $R_0$ is invertible. +\begin{center} + $R_0$ is invertible $\iff \hat{X}$ has full column rank $\iff \hat{X}^T\hat{X} \succ 0$. +\end{center} +$R_0$ is invertible and the triangular system can be solved via backsubstitution. This claim is proved in \hyperref[proofs: fullcolumn]{the last section}. + +\section{L-BFGS}\label{ch: L-BFGS} + +We can define +\begin{equation} + \begin{aligned} + g(w) = {f(w)}^2 = \norm{\hat{X}w-\hat{y}}^2 + \end{aligned} +\end{equation} +and reformulate the problem equivalently in terms of $g(w)$, since it is monotonic. +\begin{equation*} + \begin{aligned} + \min_{w}\ g(w) = \min_{w}\ \norm{\hat{X}w-\hat{y}}^2 = \min_{w}\ {\bigl(\hat{X}w - \hat{y}\bigr)}^T\bigl(\hat{X}w - \hat{y}\bigr) + \end{aligned} +\end{equation*} +The gradient of $g$ with respect to $w$ is +\begin{equation*} + \begin{aligned} + \nabla g(w) = 2\hat{X}^T\bigl(\hat{X}w - \hat{y}\bigr) + \end{aligned} +\end{equation*} + +Likewise the gradient of $f(w)$ is as follows: +\begin{equation*} + \nabla f(w) = \frac{1}{\norm{\hat{X} w - \hat{y}}} \hat{X}^T\bigl(\hat{X}w - \hat{y}\bigr) +\end{equation*} +but gives much worse performance since it is no longer quadratic. + +The function is L-smooth since $\forall w, w' \in \mathbb{R}^m,\ \text{with } w \neq w'$: + +\vspace{6pt} + +\begin{tblr}{colspec={crl}, colsep={0pt}} + & \(\norm{\nabla g(w) - \nabla g(w')}\) &\(\ \leq L \norm{w - w'}\)\\ + \(\iff\) & \(\norm{\hat{X}^T(\hat{X}w - w') - \hat{X}^T (\hat{X} w' -\hat{y})}\) & \(\ \leq L \norm{w - w'}\) \\ + \(\iff\) & \(\norm{\hat{X}^T \hat{X} (w-w')}\) & \(\ \leq L \norm{w - w'}\) \\ + \(\Longleftarrow\) & \(\norm{\hat{X}^T \hat{X}} \norm{w-w'}\) & \(\ \leq L \norm{w - w'}\) \\ + \(\iff\) & \(\norm{\hat{X}^T \hat{X}}\) & \(\ \leq L\ \) +\end{tblr} + +\vspace{6pt} + +The function $g$ is also strongly convex since \( \nabla^2g(w) = \hat{X}^T \hat{X} \succ 0\). + +The tomography of $g(w)$ with respect to the direction $p$ is: +\begin{align} + \phi(\alpha)&={(\hat{X}(w+\alpha p) - \hat{y})}^T \cdot (\hat{X}(w+\alpha p) - \hat{y}) \notag\\ + \frac{d \phi(\alpha)}{d \alpha} &= 2 w^T \hat{X}^T \hat{X} p - 2 \hat{y}^T \hat{X} p + 2 \alpha p^T \hat{X}^T \hat{X} p \notag\\ + \frac{d^2 \phi(\alpha)}{d \alpha^2} &= 2 p^T \hat{X}^T \hat{X} p \label{definitions: hessian tomography} +\end{align} + +Since $\frac{d^2 \phi(\alpha)}{d \alpha^2}$ is constant, the tomography is simply a parabola and since $\hat{X}^T \hat{X}$ is positive definite, the dot product $\langle p, p \rangle_{\hat{X}^T \hat{X}}$ is always positive and the parabola always has a minimum. The minimum is found by solving $\frac{d \phi(\alpha)}{d \alpha}$ for $0$: + +\[ \alpha_{\min} = \frac{\hat{y}^T \hat{X} p - w^T \hat{X}^T \hat{X} p}{p^T \hat{X}^T \hat{X} p} \] + +\section{Conditioning}\label{subsec:conditioning} + +We check the condition number $\kappa(\hat{X})$ when the regularization term $\lambda > 0$ varies. +\[ +\kappa(\hat{X}) = \norm{\hat{X}} \norm{\hat{X}^{T}} = \frac{\sigma_1}{\sigma_m} = \sqrt{\frac{\lambda_{\max}}{\lambda_{\min}}} +\] +with $\sigma_1, \sigma_m$ being respectively the largest and smallest singular values of $\hat{X}$ and $\lambda_{\max}, \lambda_{\min}$ being the largest and smallest eigenvalues of $\hat{X}^T\hat{X}$.\\ +Knowing that $\hat{X}^T\hat{X} = XX^T + \lambda^2I_m$, we have that +\begin{center} + \begin{tblr}{colspec={c}, colsep={0pt}, column{1} = {mode = math}} + \lambda_{max} = \lambda_1 + \lambda^2 \\ + \lambda_{min} = \lambda_m + \lambda^2 \\ + \end{tblr} +\end{center} +with $\lambda_1, \lambda_m$ being the largest and smallest eigenvalues of $XX^T$, which are translated by $\lambda^2$ as a result of adding $\lambda^2I_m$ (\autoref{proof:eigenvalues_translation})\\ +In \autoref{proofs: eigenvalues} we show that $\lambda_m = 0$ and conclude that $\kappa(\hat{X})$ scales linearly with $\frac{1}{\lambda}$: +\[ +\kappa(\hat{X}) = \sqrt{\frac{\lambda_{\max}}{\lambda_{\min}}} = \sqrt{\frac{\lambda_{1} + \lambda^2}{\lambda_{m} + \lambda^2}} = \frac{\sqrt{\lambda_{1} + \lambda^2}}{{\sqrt{\lambda^2}}} = \frac{\sqrt{\lambda_{1} + \lambda^2}}{\lambda} +\] +if $\lambda_1 > 0$. + +For lambda close to zero we have $\frac{\sqrt{\lambda_{1} + \lambda^2}}{\lambda} \approx O\left(\frac{1}{\lambda}\right)$. +This property is witnessed in \autoref{fig:condition}, which is in logarithmic scale: +\begin{figure}[htbp] + \centering + \includegraphics[width=0.7\linewidth]{(2) - problem definition/images/conditioning.png} % chktex 8 + \caption{$\kappa(\hat{X})$ \textit{for different values of} $\lambda$}\label{fig:condition} +\end{figure} + +%%% Local Variables: +%%% mode: latex +%%% TeX-master: "../main" +%%% TeX-command-extra-options: "-shell-escape" +%%% End: diff --git a/Report/(3) - algorithms/algorithms.aux b/Report/(3) - algorithms/algorithms.aux new file mode 100644 index 0000000..1506638 --- /dev/null +++ b/Report/(3) - algorithms/algorithms.aux @@ -0,0 +1,186 @@ +\relax +\providecommand\hyper@newdestlabel[2]{} +\@writefile{toc}{\contentsline {chapter}{\numberline {3}Algorithms}{6}{chapter.3}\protected@file@percent } +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loa}{\addvspace {10\p@ }} +\newlabel{ch: algorithms}{{3}{6}{Algorithms}{chapter.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.1}QR}{6}{section.3.1}\protected@file@percent } +\@writefile{loa}{\contentsline {algocf}{\numberline {1}{\ignorespaces Thin QR}}{7}{algocf.1}\protected@file@percent } +\newlabel{algo: thinQR}{{1}{7}{QR}{algocf.1}{}} +\@writefile{loa}{\contentsline {algocf}{\numberline {2}{\ignorespaces householder\_vector}}{7}{algocf.2}\protected@file@percent } +\newlabel{algo: householder_vector}{{2}{7}{QR}{algocf.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.2}L-BFGS}{8}{section.3.2}\protected@file@percent } +\@writefile{loa}{\contentsline {algocf}{\numberline {3}{\ignorespaces Limited Memory BFGS}}{8}{algocf.3}\protected@file@percent } +\newlabel{algo: L-BFGS}{{3}{8}{L-BFGS}{algocf.3}{}} +\@writefile{loa}{\contentsline {algocf}{\numberline {4}{\ignorespaces Limited Memory BFGS {-} Two-Loop Recursion}}{9}{algocf.4}\protected@file@percent } +\newlabel{algo: L-BFGS Two-Loop Recursion}{{4}{9}{L-BFGS}{algocf.4}{}} +\newlabel{algo: convergence1}{{1}{9}{}{Item.1}{}} +\newlabel{algo: convergence2}{{2}{9}{}{Item.2}{}} +\newlabel{algo: convergence3}{{3}{9}{}{Item.3}{}} +\newlabel{eq:6}{{3}{9}{}{Item.3}{}} +\newlabel{algo: definition y_k}{{3.1}{10}{}{equation.3.2.1}{}} +\@writefile{loe}{\addvspace {10\p@ }} +\@writefile{loe}{\contentsline {mtheo}{\ifthmt@listswap \else \numberline {\let \autodot \@empty }\fi Theorem}{10}{thmt@dummyctr.dummy.1}\protected@file@percent } +\@writefile{loe}{\contentsline {mproof}{\ifthmt@listswap \else \numberline {\let \autodot \@empty }\fi Proof}{10}{thmt@dummyctr.dummy.2}\protected@file@percent } +\@setckpt{(3) - algorithms/algorithms}{ +\setcounter{page}{12} +\setcounter{equation}{1} +\setcounter{enumi}{3} +\setcounter{enumii}{0} +\setcounter{enumiii}{0} +\setcounter{enumiv}{0} +\setcounter{footnote}{0} +\setcounter{mpfootnote}{0} +\setcounter{part}{0} +\setcounter{chapter}{3} +\setcounter{section}{2} +\setcounter{subsection}{0} +\setcounter{subsubsection}{0} +\setcounter{paragraph}{0} +\setcounter{subparagraph}{0} +\setcounter{figure}{0} +\setcounter{table}{0} +\setcounter{tabx@nest}{0} +\setcounter{listtotal}{0} +\setcounter{listcount}{0} +\setcounter{liststart}{0} +\setcounter{liststop}{0} +\setcounter{citecount}{0} +\setcounter{citetotal}{0} +\setcounter{multicitecount}{0} +\setcounter{multicitetotal}{0} +\setcounter{instcount}{2} +\setcounter{maxnames}{3} +\setcounter{minnames}{3} +\setcounter{maxitems}{3} +\setcounter{minitems}{1} +\setcounter{citecounter}{0} +\setcounter{maxcitecounter}{0} +\setcounter{savedcitecounter}{0} +\setcounter{uniquelist}{0} +\setcounter{uniquename}{0} +\setcounter{refsection}{0} +\setcounter{refsegment}{0} +\setcounter{maxextratitle}{0} +\setcounter{maxextratitleyear}{0} +\setcounter{maxextraname}{0} +\setcounter{maxextradate}{0} +\setcounter{maxextraalpha}{0} +\setcounter{abbrvpenalty}{50} +\setcounter{highnamepenalty}{50} +\setcounter{lownamepenalty}{25} +\setcounter{maxparens}{3} +\setcounter{parenlevel}{0} +\setcounter{blx@maxsection}{0} +\setcounter{mincomprange}{10} +\setcounter{maxcomprange}{100000} +\setcounter{mincompwidth}{1} +\setcounter{afterword}{0} +\setcounter{savedafterword}{0} +\setcounter{annotator}{0} +\setcounter{savedannotator}{0} +\setcounter{author}{0} +\setcounter{savedauthor}{0} +\setcounter{bookauthor}{0} +\setcounter{savedbookauthor}{0} +\setcounter{commentator}{0} +\setcounter{savedcommentator}{0} +\setcounter{editor}{0} +\setcounter{savededitor}{0} +\setcounter{editora}{0} +\setcounter{savededitora}{0} +\setcounter{editorb}{0} +\setcounter{savededitorb}{0} +\setcounter{editorc}{0} +\setcounter{savededitorc}{0} +\setcounter{foreword}{0} +\setcounter{savedforeword}{0} +\setcounter{holder}{0} +\setcounter{savedholder}{0} +\setcounter{introduction}{0} +\setcounter{savedintroduction}{0} +\setcounter{namea}{0} +\setcounter{savednamea}{0} +\setcounter{nameb}{0} +\setcounter{savednameb}{0} +\setcounter{namec}{0} +\setcounter{savednamec}{0} +\setcounter{translator}{0} +\setcounter{savedtranslator}{0} +\setcounter{shortauthor}{0} +\setcounter{savedshortauthor}{0} +\setcounter{shorteditor}{0} +\setcounter{savedshorteditor}{0} +\setcounter{labelname}{0} +\setcounter{savedlabelname}{0} +\setcounter{institution}{0} +\setcounter{savedinstitution}{0} +\setcounter{lista}{0} +\setcounter{savedlista}{0} +\setcounter{listb}{0} +\setcounter{savedlistb}{0} +\setcounter{listc}{0} +\setcounter{savedlistc}{0} +\setcounter{listd}{0} +\setcounter{savedlistd}{0} +\setcounter{liste}{0} +\setcounter{savedliste}{0} +\setcounter{listf}{0} +\setcounter{savedlistf}{0} +\setcounter{location}{0} +\setcounter{savedlocation}{0} +\setcounter{organization}{0} +\setcounter{savedorganization}{0} +\setcounter{origlocation}{0} +\setcounter{savedoriglocation}{0} +\setcounter{origpublisher}{0} +\setcounter{savedorigpublisher}{0} +\setcounter{publisher}{0} +\setcounter{savedpublisher}{0} +\setcounter{language}{0} +\setcounter{savedlanguage}{0} +\setcounter{origlanguage}{0} +\setcounter{savedoriglanguage}{0} +\setcounter{pageref}{0} +\setcounter{savedpageref}{0} +\setcounter{textcitecount}{0} +\setcounter{textcitetotal}{0} +\setcounter{textcitemaxnames}{0} +\setcounter{biburlbigbreakpenalty}{100} +\setcounter{biburlbreakpenalty}{200} +\setcounter{biburlnumpenalty}{0} +\setcounter{biburlucpenalty}{0} +\setcounter{biburllcpenalty}{0} +\setcounter{smartand}{1} +\setcounter{bbx:relatedcount}{0} +\setcounter{bbx:relatedtotal}{0} +\setcounter{cbx@tempcnta}{0} +\setcounter{cbx@tempcntb}{1} +\setcounter{cbx@tempcntc}{0} +\setcounter{cbx@tempcntd}{-1} +\setcounter{float@type}{4} +\setcounter{parentequation}{0} +\setcounter{ALG@line}{0} +\setcounter{ALG@rem}{0} +\setcounter{ALG@nested}{0} +\setcounter{ALG@Lnr}{2} +\setcounter{ALG@blocknr}{10} +\setcounter{ALG@storecount}{0} +\setcounter{ALG@tmpcounter}{0} +\setcounter{thmt@dummyctr}{2} +\setcounter{nlinenum}{0} +\setcounter{caption@flags}{2} +\setcounter{continuedfloat}{0} +\setcounter{subfigure}{0} +\setcounter{subtable}{0} +\setcounter{section@level}{0} +\setcounter{Item}{3} +\setcounter{Hfootnote}{0} +\setcounter{bookmark@seq@number}{0} +\setcounter{AlgoLine}{11} +\setcounter{algocfline}{4} +\setcounter{algocfproc}{4} +\setcounter{algocf}{4} +\setcounter{mlemma}{0} +} diff --git a/Report/(3) - algorithms/algorithms.tex b/Report/(3) - algorithms/algorithms.tex new file mode 100644 index 0000000..31921b3 --- /dev/null +++ b/Report/(3) - algorithms/algorithms.tex @@ -0,0 +1,218 @@ +% chktex-file 9 chktex-file 17 +\chapter{Algorithms}\label{ch: algorithms} + +\section{QR} +The algorithm has been implemented considering that the input matrix $A \in \mathbb{R}^{m \times n}$, where $m$ may be different from $n$, namely it can be rectangular \textit{horizontally} or \textit{vertically}. In this version we store in a proper data structure a matrix $\Upsilon \in {m \times n}$ of the following form ($m > n$ in this example): + +\begin{equation*} + \Upsilon = {(\upsilon_{i,j})}_{i,j} = \begin{tikzpicture}[baseline=-1ex] + \matrix[% + matrix of math nodes, + nodes in empty cells, + left delimiter={[},right delimiter={]}, + inner xsep=2pt, column sep=6pt, + ] (m) + {% + \vphantom{1} & * & \cdots & * \\ + & \vphantom{1} & \ddots & \vdots \\ + & & \vphantom{1} & * \\ + & & & \vphantom{1} \\ + u_1 & u_2 & \cdots & u_n \\ + \vphantom{1} & \vphantom{1} & \vphantom{1} & \vphantom{1} \\ + }; + \node[rectangle, draw, fit={(m-1-1) (m-6-1)}, inner sep=-1.5pt, text width=22pt] {}; + \node[rectangle, draw, fit={(m-2-2) (m-6-2)}, inner sep=-1.5pt, text width=22pt] {}; + \node[rectangle, draw, fit={(m-3-3) (m-6-3)}, inner sep=-1.5pt, text width=22pt] {}; + \node[rectangle, draw, fit={(m-4-4) (m-6-4)}, inner sep=-1.5pt, text width=22pt] {}; + \end{tikzpicture} +\end{equation*} + +\begin{center} + $u_k \in \mathbb{R}^{m - k + 1},\ 1 \leq k \leq n$ +\end{center} +and the values of the diagonal of R in a vector $d \in \mathbb{R}^{n}$. The $*$ entries are elements computed in the QR factorization belonging to the upper triangular matrix, yielded by line 6 of \hyperref[algo: thinQR]{Algorithm 1}. In this way we are allowed to lazily perform the products $Qy$ and $Q^T y$ by means of the householder vectors $u_1 \dots, u_n $ that we stored. On the other hand, to compute a product between the upper part of $\Upsilon$ and an input vector we reconstruct the upper triangular matrix by taking element $\upsilon_{ij} \text{ such that } j > i$ and attach the vector $d$ as the diagonal of the resulting matrix. +The zeros of the matrix $R$ are ignored. + +\begin{algorithm}[H] + \SetAlgoLined% + \caption{Thin QR}\label{algo: thinQR} + \SetKwInOut{Input}{Input} + \SetKwInOut{Output}{Output} + + \BlankLine% + \Input{$A \in \mathbb{R}^{m \times n}$} + \Output{$Q \in \mathbb{R}^{m \times m},\ R \in \mathbb{R}^{m \times n}$ implicit $QR$ factorization of $A$} + + \BlankLine% + $\Upsilon = copy(A)$ \\ + $d = zeros(\min(m, n))$ \\ + + \For{ + $k \in 1 \dots \min(m, n)$ + }{ + $u_k, s_k = householder\_vector(\Upsilon[k:m, k])$\\ + $d_k = s_k$ \\ + $\Upsilon[k:m, k+1:n] = \Upsilon[k:m, k+1:n] - 2u(u^T \Upsilon[k:m, k+1:n])$\\ + $\Upsilon[k:m, k] = u_k$ + } + + \Return$\Upsilon, d$ +\end{algorithm} + +\begin{algorithm}[H] + \SetAlgoLined% + \caption{householder\_vector}\label{algo: householder_vector} + \SetKwInOut{Input}{Input} + \SetKwInOut{Output}{Output} + + \BlankLine% + \Input{$x \in \mathbb{R}^d$} + \Output{$u \in \mathbb{R}^{d},\ s \in \mathbb{R}$ householder vector of $x$} + + \BlankLine% + $s = \norm{x}$ \\ + \If{$x_1 \geq 0$}{ + $s = -s$ + } + $u = copy(x)$ \\ + $u_1 = u_1 - s$ \\ + $u = u\ / \norm{u}$ \\ + + \Return$u, s$ +\end{algorithm} + +We assume $m > n$ as the case $n > m$ is similar for the complexity analysis. The time complexity of this algorithm is $\theta\bigl(mn^2 \bigr) \approx \theta\bigl(n^3 \bigr)$, because $m \approx n$ in (P). We will see in \hyperref[ch: experiments]{section Experiments} that the running time scales linearly with $m$ as expected, where $m$ is the size of $\hat{X}$. + +\newpage +\section{L-BFGS} + +We follow the syntax from \textit{Numerical Optimization}\cite{Numerical-Optimization-2006} and define $f_k = f(x_k)$ + +\begin{algorithm}[H] + \SetAlgoLined% + \caption{Limited Memory BFGS}\label{algo: L-BFGS} + \SetKwInOut{Input}{Input} + \SetKwInOut{Output}{Output} + + \BlankLine% + \Input{$\textbf{f}: \mathbb{R}^n \longrightarrow \mathbb{R},\ \textbf{x} \in \mathbb{R}^n,\ m \text{ memory, } \epsilon \text{ tolerance}$} + \Output{${\bf x^*}\ \text{ending point},\ {\bf f(x^*)},\ {\bf \nabla f(x^*)}$} + + \BlankLine% + $k = 0$ \\ + \While{$\nabla f_k \geq \epsilon \nabla f_0$} { + \uIf{storage is empty}{ + $H_k^0 = I$ + }\uElse{ + $H_k^0 = \frac{\langle y_{k-1}, s_{k-1} \rangle}{\norm{y_{k-1}}^2} \cdot I$ + } + Calculate $p_k = H_k \nabla{f_k}$ with \hyperref[algo: L-BFGS Two-Loop Recursion]{\textbf{Algorithm 4}} \\ + Choose $\alpha_k$ satisfying the Armijo-Wolfe conditions or with exact line search \\ + $x_{k+1} = x_k + \alpha_k p_k$ \\ + $s_k = x_{k+1} - x_k$ \\ + $y_k = \nabla f_{k+1} - \nabla f_k$ \\ + $curvature = \langle y_k, s_k \rangle$ \\ + $\rho_k = curvature^{-1}$ \\ + \uIf{$curvature \leq 10^{-16}$}{ + free the storage and start again from gradient descent + }\uElse{ + Discard the vector pair $\{s_{k-m}, y_{k-m}, \rho_{k-m}\}$ from storage \\ + Save $s_k, y_k, \rho_k$ + } + + $k = k + 1$ + } + \Return$x_k$, $f_k$, $\nabla f_k$ +\end{algorithm} +\begin{algorithm}[H] + \SetAlgoLined% + \caption{Limited Memory BFGS {-} Two-Loop Recursion}\label{algo: L-BFGS Two-Loop Recursion} + + $q = \nabla f_k$ \\ + \For{$i = (k - 1), \dots, (k - m)$}{ + $\alpha_i = \rho_i s_i^T q$ \\ + $q = q - \alpha_i y_i$ \\ + } + + $r = H_k^0 q$ \\ + \For{$i = (k - m), \dots, (k - 1)$}{ + $\beta = \rho_i y_i^T r$ \\ + $r = r + s_i\bigl(\alpha_i - \beta\bigr)$ \\ + } + + \Return$-r$ + +\end{algorithm} +In our implementation we keep the triplets $(s_k, y_k, \rho_k)$ in a circular buffer with capacity $m$ and the values of $\alpha_i$ in \hyperref[algo: L-BFGS Two-Loop Recursion]{Algorithm 4} in a stack such that no explicit indices are needed. + +In case the curvature of the function is too small, we free the storage and restart with a gradient step. + +We prefer using an exact line search to compute the step size over an inexact line search since the computational cost for our problem is lesser. + +\subsection*{Convergence} +To prove that the implemented method converges to the global minimum of the function we have to optimize, we follow~\cite{convergence_lbfgs} and state the following assumptions about our problem: +\begin{enumerate} + \item\label{algo: convergence1} $f \in C^2$ + \item\label{algo: convergence2} The level sets $\mathcal{L} = \{ x \in \mathbb{R}^n\ |\ f(x) \leq f(x_0) \} $ is convex + \item\label{algo: convergence3} $\exists\ M_1, M_2 \in \mathbb{R}^+$ such that + \begin{equation*} + M_1\norm{z}^2 \leq z^T G(x) z \leq M_2\norm{z}^2\label{eq:6} + \end{equation*} + $\forall z \in \mathbb{R}^n$ and $\forall x \in \mathcal{L}$ +\end{enumerate} + +We follow the publication's notation and define: + +\[ G(x) \coloneqq \nabla^{2}f(x) \] +\[ \bar{G}_k(x) \coloneqq \int_0^1 G(x_k + \tau \alpha_k p_k) d\tau \] + +From Taylor's theorem: + +\begin{equation}\label{algo: definition y_k} + y_k = \bar{G}_k \alpha_k p_k = \bar{G}_k s_k +\end{equation} + +The first assumption for our problem follows from the definition. The second assumption is proved by \autoref{definitions: hessian tomography}. The third assumption is also a consequence of the fact that the hessian of $f$ is constant. + +% \[ z_k \coloneqq {\bar{G}_k}^{1/2} s_k \] + +\begin{mtheo} + Let $B_0$ be any symmetric positive definite initial matrix, and let $x_0$ be a starting point for which the Assumptions~\ref{algo: convergence1},~\ref{algo: convergence2} and~\ref{algo: convergence3} hold, then the sequence ${x_k}$ generated by the L-BFGS algorithm converges to the minimizer $x^*$ of $f$ linearly. +\end{mtheo} + +\begin{mproof} + Using \autoref{algo: definition y_k} and Assumption~\ref{algo: convergence3}: + \[ M_1 \norm{s_k}^2 \leq y_k^T s_k \leq M_2 \norm{s_k}^2 \] + and: + \[ \frac{\norm{y_k}^2}{y_k^T s_k} = \frac{s_k^T \hat{G}_k^2 s_k}{s_k^T \hat{G}_k s_k} \] + Both trace and determinant can be expressed in terms of the trace and determinant of the starting matrix from which the approximate hessian is constructed: + \begin{align*} + \Tr(B_{k+1}) &\leq \Tr(B_k^{(0)}) + \Tilde{m} M_2 \leq M_3 \\ + \det(B_{k+1}) &= \det(B_k^{(0)}) \cdot \prod_{l=0}^{\Tilde{m}-1} \frac{y_l^T s_l}{s_l^T B_k^{(l)} s_l} \geq \det\left(B_k^{(0)} {\left(\frac{M_1}{M_3}\right)}^{\Tilde{m}}\right) \geq M_4 + \end{align*} + where $\Tilde{m}$ is the memory size and $M_3$ and $M_4$ are chosen appropriately in $\mathbb{R}^+$. + + From these two bounds we have that for some constant $\delta > 0$: + + \[ \cos(\theta_k) = \frac{s_k^T B_k s_k}{\norm{s_k} \norm{B_k s_k}} \geq \delta \] + Since with exact line search the Armijo condition $f(x_k + \alpha_k p_k) \leq f(x_k) + m_1 \alpha_k \nabla f(x_k)$ is always satisfied if the constant $m_1$ does not exclude the minimum $x_*$ and since the strong Wolfe condition $\norm{\nabla f(x_k + \alpha_k p_k)} \leq m_3 \norm{\nabla f(x_k)}$ is also always satisfied since $\norm{\nabla f(x_k + \alpha_k p_k)} = O(u)$, follows from the two conditions and Assumptions~\ref{algo: convergence1} and~\ref{algo: convergence2} that: + \begin{align*} + & f(x_{k+1}) - f(x_*) \leq (1 - c \cos^2(\theta_k) (f(x_k) - f(x_*))) \\ + \implies& f(x_k) - f(x_*) \leq {(1 - c \cdot \delta^2)}^k (f(x_0) - f(x_*)) \\ + \implies& f(x_k) - f(x_*) \leq r^k (f(x_0) - f(x_*)) + \end{align*} + for some $r \in [0, 1)$. Using Assumption~\ref{algo: convergence3}: + \begin{gather*} + \frac{1}{2} M_1 \norm{x_k - x_*}^2 \leq f(x_k) - f(x_*) \\ + \implies \norm{x_k - x_*} \leq r^{k/2} {\left( 2 \frac{f(x_0) - f(x_*)}{M_1} \right)}^{(1/2)} + \end{gather*} + so the sequence $\{x_k\}$ is linearly convergent. +\end{mproof} + +The implementation of L-BFGS that uses Armijo-Wolfe line search also satisfies the assumptions so it also converges linearly to $x_*$. + +%%% Local Variables: +%%% mode: latex +%%% TeX-master: "../main" +%%% TeX-command-extra-options: "-shell-escape" +%%% End: diff --git a/Report/(4) - experiments/experiments.aux b/Report/(4) - experiments/experiments.aux new file mode 100644 index 0000000..c6adbe2 --- /dev/null +++ b/Report/(4) - experiments/experiments.aux @@ -0,0 +1,221 @@ +\relax +\providecommand\hyper@newdestlabel[2]{} +\@writefile{toc}{\contentsline {chapter}{\numberline {4}Experiments}{12}{chapter.4}\protected@file@percent } +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loa}{\addvspace {10\p@ }} +\newlabel{ch: experiments}{{4}{12}{Experiments}{chapter.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.1}QR}{13}{section.4.1}\protected@file@percent } +\newlabel{fig:QR-error-lambda}{{4.1a}{13}{\textit {QR decomposition errors and backward stability for different} $\lambda $}{figure.caption.3}{}} +\newlabel{sub@fig:QR-error-lambda}{{a}{13}{\textit {QR decomposition errors and backward stability for different} $\lambda $}{figure.caption.3}{}} +\newlabel{fig:QR-forward}{{4.1b}{13}{\textit {QR factorization forward stability on Q and R for different} $\lambda $}{figure.caption.3}{}} +\newlabel{sub@fig:QR-forward}{{b}{13}{\textit {QR factorization forward stability on Q and R for different} $\lambda $}{figure.caption.3}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.1}{\ignorespaces \textit {Errors and scalability of the QR decomposition for different values of} $\lambda $}}{13}{figure.caption.3}\protected@file@percent } +\newlabel{fig:qrtests}{{4.1}{13}{\textit {Errors and scalability of the QR decomposition for different values of} $\lambda $}{figure.caption.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.2}L-BFGS}{14}{section.4.2}\protected@file@percent } +\newlabel{fig:gradnorm-res-rel-ill}{{4.2a}{14}{\textit {Ill-conditioned matrix}}{figure.caption.4}{}} +\newlabel{sub@fig:gradnorm-res-rel-ill}{{a}{14}{\textit {Ill-conditioned matrix}}{figure.caption.4}{}} +\newlabel{fig:gradnorm-res-rel-wellll}{{4.2b}{14}{\textit {Well-conditioned matrix}}{figure.caption.4}{}} +\newlabel{sub@fig:gradnorm-res-rel-wellll}{{b}{14}{\textit {Well-conditioned matrix}}{figure.caption.4}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.2}{\ignorespaces $\left \lVert \nabla f\right \rVert $\textit {, Residual, Relative Error of L-BFGS execution on ill and well-conditioned matrices}}}{14}{figure.caption.4}\protected@file@percent } +\newlabel{fig:gradnorm-res-rel}{{4.2}{14}{$\norm {\nabla f}$\textit {, Residual, Relative Error of L-BFGS execution on ill and well-conditioned matrices}}{figure.caption.4}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.3}{\ignorespaces \textit {Line Search algorithms comparison}}}{15}{figure.caption.5}\protected@file@percent } +\newlabel{fig:LS-comparison}{{4.3}{15}{\textit {Line Search algorithms comparison}}{figure.caption.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.3}Comparison between QR and L-BFGS}{15}{section.4.3}\protected@file@percent } +\newlabel{fig:time-comparison-illcond-n}{{4.4a}{16}{\textit {Ill-conditioned matrix}}{figure.caption.6}{}} +\newlabel{sub@fig:time-comparison-illcond-n}{{a}{16}{\textit {Ill-conditioned matrix}}{figure.caption.6}{}} +\newlabel{fig:time-comparison-wellcond-n}{{4.4b}{16}{\textit {Well-conditioned matrix}}{figure.caption.6}{}} +\newlabel{sub@fig:time-comparison-wellcond-n}{{b}{16}{\textit {Well-conditioned matrix}}{figure.caption.6}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.4}{\ignorespaces \textit {Time and Memory scalability comparison of QR and L-BFGS on ill and well-conditioned matrices, varying \textbf {n}}}}{16}{figure.caption.6}\protected@file@percent } +\newlabel{fig:QRvsLBFGS-time-comparison-n}{{4.4}{16}{\textit {Time and Memory scalability comparison of QR and L-BFGS on ill and well-conditioned matrices, varying \textbf {n}}}{figure.caption.6}{}} +\newlabel{fig:time-comparison-illcond-m}{{4.5a}{16}{\textit {Ill-conditioned matrix}}{figure.caption.7}{}} +\newlabel{sub@fig:time-comparison-illcond-m}{{a}{16}{\textit {Ill-conditioned matrix}}{figure.caption.7}{}} +\newlabel{fig:time-comparison-wellcond-m}{{4.5b}{16}{\textit {Well-conditioned matrix}}{figure.caption.7}{}} +\newlabel{sub@fig:time-comparison-wellcond-m}{{b}{16}{\textit {Well-conditioned matrix}}{figure.caption.7}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.5}{\ignorespaces \textit {Time and Memory scalability comparison of QR and L-BFGS on ill and well-conditioned matrices, varying \textbf {m}}}}{16}{figure.caption.7}\protected@file@percent } +\newlabel{fig:QRvsLBFGS-time-comparison-m}{{4.5}{16}{\textit {Time and Memory scalability comparison of QR and L-BFGS on ill and well-conditioned matrices, varying \textbf {m}}}{figure.caption.7}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.4}Other Experiments}{17}{section.4.4}\protected@file@percent } +\newlabel{sec:other_experiments}{{4.4}{17}{Other Experiments}{section.4.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {4.4.1}The Effect of the Memory Size}{17}{subsection.4.4.1}\protected@file@percent } +\newlabel{fig:error-memory-size-illcond}{{4.6a}{17}{\textit {Ill-conditioned matrix}}{figure.caption.8}{}} +\newlabel{sub@fig:error-memory-size-illcond}{{a}{17}{\textit {Ill-conditioned matrix}}{figure.caption.8}{}} +\newlabel{fig:error-memory-size-wellcond}{{4.6b}{17}{\textit {Well-conditioned matrix}}{figure.caption.8}{}} +\newlabel{sub@fig:error-memory-size-wellcond}{{b}{17}{\textit {Well-conditioned matrix}}{figure.caption.8}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.6}{\ignorespaces \textit {The effect of the memory size}}}{17}{figure.caption.8}\protected@file@percent } +\newlabel{fig:error-memory-size}{{4.6}{17}{\textit {The effect of the memory size}}{figure.caption.8}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {4.4.2}A Comparison of Quasi-Newton Methods}{18}{subsection.4.4.2}\protected@file@percent } +\newlabel{fig:BFGSvsLBFGS-rate}{{4.7a}{19}{\textit {Convergence rate}}{figure.caption.9}{}} +\newlabel{sub@fig:BFGSvsLBFGS-rate}{{a}{19}{\textit {Convergence rate}}{figure.caption.9}{}} +\newlabel{fig:BFGSvsLBFGS-scalability}{{4.7b}{19}{\textit {Time and memory scalability}}{figure.caption.9}{}} +\newlabel{sub@fig:BFGSvsLBFGS-scalability}{{b}{19}{\textit {Time and memory scalability}}{figure.caption.9}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.7}{\ignorespaces \textit {BFGS vs L-BFGS}}}{19}{figure.caption.9}\protected@file@percent } +\newlabel{fig:BFGSvsLBFGS-comparison}{{4.7}{19}{\textit {BFGS vs L-BFGS}}{figure.caption.9}{}} +\newlabel{fig:Quasi-newton-dogleg-time}{{4.8a}{20}{\textit {With Dogleg}}{figure.caption.10}{}} +\newlabel{sub@fig:Quasi-newton-dogleg-time}{{a}{20}{\textit {With Dogleg}}{figure.caption.10}{}} +\newlabel{fig:Quasi-newton-no-dogleg-time}{{4.8b}{20}{\textit {Without Dogleg}}{figure.caption.10}{}} +\newlabel{sub@fig:Quasi-newton-no-dogleg-time}{{b}{20}{\textit {Without Dogleg}}{figure.caption.10}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.8}{\ignorespaces \textit {Quasi-Newton methods running time comparison}}}{20}{figure.caption.10}\protected@file@percent } +\newlabel{fig:Quasi-newton-time}{{4.8}{20}{\textit {Quasi-Newton methods running time comparison}}{figure.caption.10}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.9}{\ignorespaces \textit {Memory allocation of the different Quasi-Newton methods}}}{20}{figure.caption.11}\protected@file@percent } +\newlabel{fig:Quasi newton comparison memory}{{4.9}{20}{\textit {Memory allocation of the different Quasi-Newton methods}}{figure.caption.11}{}} +\@setckpt{(4) - experiments/experiments}{ +\setcounter{page}{21} +\setcounter{equation}{0} +\setcounter{enumi}{3} +\setcounter{enumii}{0} +\setcounter{enumiii}{0} +\setcounter{enumiv}{0} +\setcounter{footnote}{0} +\setcounter{mpfootnote}{0} +\setcounter{part}{0} +\setcounter{chapter}{4} +\setcounter{section}{4} +\setcounter{subsection}{2} +\setcounter{subsubsection}{0} +\setcounter{paragraph}{0} +\setcounter{subparagraph}{0} +\setcounter{figure}{9} +\setcounter{table}{0} +\setcounter{tabx@nest}{0} +\setcounter{listtotal}{0} +\setcounter{listcount}{0} +\setcounter{liststart}{0} +\setcounter{liststop}{0} +\setcounter{citecount}{0} +\setcounter{citetotal}{0} +\setcounter{multicitecount}{0} +\setcounter{multicitetotal}{0} +\setcounter{instcount}{5} +\setcounter{maxnames}{3} +\setcounter{minnames}{3} +\setcounter{maxitems}{3} +\setcounter{minitems}{1} +\setcounter{citecounter}{0} +\setcounter{maxcitecounter}{0} +\setcounter{savedcitecounter}{0} +\setcounter{uniquelist}{0} +\setcounter{uniquename}{0} +\setcounter{refsection}{0} +\setcounter{refsegment}{0} +\setcounter{maxextratitle}{0} +\setcounter{maxextratitleyear}{0} +\setcounter{maxextraname}{0} +\setcounter{maxextradate}{0} +\setcounter{maxextraalpha}{0} +\setcounter{abbrvpenalty}{50} +\setcounter{highnamepenalty}{50} +\setcounter{lownamepenalty}{25} +\setcounter{maxparens}{3} +\setcounter{parenlevel}{0} +\setcounter{blx@maxsection}{0} +\setcounter{mincomprange}{10} +\setcounter{maxcomprange}{100000} +\setcounter{mincompwidth}{1} +\setcounter{afterword}{0} +\setcounter{savedafterword}{0} +\setcounter{annotator}{0} +\setcounter{savedannotator}{0} +\setcounter{author}{0} +\setcounter{savedauthor}{0} +\setcounter{bookauthor}{0} +\setcounter{savedbookauthor}{0} +\setcounter{commentator}{0} +\setcounter{savedcommentator}{0} +\setcounter{editor}{0} +\setcounter{savededitor}{0} +\setcounter{editora}{0} +\setcounter{savededitora}{0} +\setcounter{editorb}{0} +\setcounter{savededitorb}{0} +\setcounter{editorc}{0} +\setcounter{savededitorc}{0} +\setcounter{foreword}{0} +\setcounter{savedforeword}{0} +\setcounter{holder}{0} +\setcounter{savedholder}{0} +\setcounter{introduction}{0} +\setcounter{savedintroduction}{0} +\setcounter{namea}{0} +\setcounter{savednamea}{0} +\setcounter{nameb}{0} +\setcounter{savednameb}{0} +\setcounter{namec}{0} +\setcounter{savednamec}{0} +\setcounter{translator}{0} +\setcounter{savedtranslator}{0} +\setcounter{shortauthor}{0} +\setcounter{savedshortauthor}{0} +\setcounter{shorteditor}{0} +\setcounter{savedshorteditor}{0} +\setcounter{labelname}{0} +\setcounter{savedlabelname}{0} +\setcounter{institution}{0} +\setcounter{savedinstitution}{0} +\setcounter{lista}{0} +\setcounter{savedlista}{0} +\setcounter{listb}{0} +\setcounter{savedlistb}{0} +\setcounter{listc}{0} +\setcounter{savedlistc}{0} +\setcounter{listd}{0} +\setcounter{savedlistd}{0} +\setcounter{liste}{0} +\setcounter{savedliste}{0} +\setcounter{listf}{0} +\setcounter{savedlistf}{0} +\setcounter{location}{0} +\setcounter{savedlocation}{0} +\setcounter{organization}{0} +\setcounter{savedorganization}{0} +\setcounter{origlocation}{0} +\setcounter{savedoriglocation}{0} +\setcounter{origpublisher}{0} +\setcounter{savedorigpublisher}{0} +\setcounter{publisher}{0} +\setcounter{savedpublisher}{0} +\setcounter{language}{0} +\setcounter{savedlanguage}{0} +\setcounter{origlanguage}{0} +\setcounter{savedoriglanguage}{0} +\setcounter{pageref}{0} +\setcounter{savedpageref}{0} +\setcounter{textcitecount}{0} +\setcounter{textcitetotal}{0} +\setcounter{textcitemaxnames}{0} +\setcounter{biburlbigbreakpenalty}{100} +\setcounter{biburlbreakpenalty}{200} +\setcounter{biburlnumpenalty}{0} +\setcounter{biburlucpenalty}{0} +\setcounter{biburllcpenalty}{0} +\setcounter{smartand}{1} +\setcounter{bbx:relatedcount}{0} +\setcounter{bbx:relatedtotal}{0} +\setcounter{cbx@tempcnta}{0} +\setcounter{cbx@tempcntb}{2} +\setcounter{cbx@tempcntc}{0} +\setcounter{cbx@tempcntd}{-1} +\setcounter{float@type}{4} +\setcounter{parentequation}{0} +\setcounter{ALG@line}{0} +\setcounter{ALG@rem}{0} +\setcounter{ALG@nested}{0} +\setcounter{ALG@Lnr}{2} +\setcounter{ALG@blocknr}{10} +\setcounter{ALG@storecount}{0} +\setcounter{ALG@tmpcounter}{0} +\setcounter{thmt@dummyctr}{2} +\setcounter{nlinenum}{0} +\setcounter{caption@flags}{2} +\setcounter{continuedfloat}{0} +\setcounter{subfigure}{0} +\setcounter{subtable}{0} +\setcounter{section@level}{0} +\setcounter{Item}{3} +\setcounter{Hfootnote}{0} +\setcounter{bookmark@seq@number}{0} +\setcounter{AlgoLine}{11} +\setcounter{algocfline}{4} +\setcounter{algocfproc}{4} +\setcounter{algocf}{4} +\setcounter{mlemma}{0} +} diff --git a/Report/(4) - experiments/experiments.tex b/Report/(4) - experiments/experiments.tex new file mode 100644 index 0000000..b2e73e4 --- /dev/null +++ b/Report/(4) - experiments/experiments.tex @@ -0,0 +1,220 @@ +\chapter{Experiments}\label{ch: experiments} +In this chapter we present the results of the experiments run on both algorithms as well as a comparison of the two methods in terms of accuracy and time scalability. + +To test the behaviour of the two methods we handle both cases in which the matrix $\hat{X}$ is well-conditioned, with $\kappa(\hat{X}) \approx 5$, and ill-conditioned, with $\kappa(\hat{X}) \approx 5 \times 10^5$. To accomplish so, we randomly generated the matrix $X$ forcing its values to be in the range $[-1, 1]$, the dimensions $m = 1000$ and $n = 20$ (except for time and memory scalability tests), and, as we have seen in \autoref{subsec:conditioning}, since we can control the conditioning directly with the \textit{hyperparameter} $\lambda$, we choose for the first case $\lambda = 10^{-4}$ and for the latter $\lambda = 10^{-12}$. + +For the QR factorization we check how the relative error and residual change with respect to different values of $\lambda$. Then, we confirm the backward stability of the decomposition over different values of $\lambda$ and check its forward stability as well. + +For what concerns L-BFGS we fix the relative tolerance $\epsilon = 10^{-14}$, the memory size $k = 7$ and the maximum number of function evaluations to $200$, knowing that the function we have to optimize can be easily optimized by the method. + +The last kind of tests we present concerns the scalability of the methods in terms of time and memory, which has been compared by modifying the matrix $\hat{X} \in \mathbb{R}^{(m+n) \times m}$ by generating random matrices with increasing $m$ and $n$ separately. As mentioned before, we brought this experiment to the case in which $\hat{X}$ is either ill-conditioned or well-conditioned. In the case of the thin-QR factorization we expect a linear dependency between the number of rows and the time needed to converge to the optimal solution, assuming a fixed number of columns. If instead we vary the number of columns we expect a quadratic dependency. + +In \autoref{sec:other_experiments} we first explore better the effect of the memory size for L-BFGS and then we provide a deeper comparison of other Quasi-Newton methods we manually implemented (even if not really required by the project instructions). + +All tests have been executed with the benchmark library \texttt{BenchmarkTools.jl}\cite{BenchmarkTools} which ignores startup and compilation time and repeated 10 times in order to get accurate estimates. + + +\section{QR} +Since we know from theory that the QR decomposition is backward stable, we expect that $\frac{\norm{\hat{X} - Q R}}{\norm{\hat{X}}} \approx u$. +Or more explicitly that for $QR = \hat{X} + \delta \hat{X}$, $\frac{\norm{\delta \hat{X}}}{\norm{\hat{X}}} = O(u)$. The results in \autoref{fig:QR-error-lambda} show a decreasing trend for relative error and residual when increasing $\lambda$ and hence decreasing the condition number $\kappa(\hat{X})$. The errors are acceptable even for the smallest lambda: $\lambda = 10^{-16}$, $\kappa(\hat{X}) \approx 5 \times 10^{15}$. The algorithm is backward stable as well, as it can be noticed from the green part of the plot.\\ +To check the forward error we QR-decomposed the original matrix $\hat{X}$ to get $Q$ and $R$ and then we perturbed it with a random matrix multiplied by a factor $\delta = 10^{-10}$. Then, we ran another QR-decomposition on the perturbed version of $\hat{X}$ to get the factors $\tilde{Q}$ and $\tilde{R}$. Finally, we evaluated $\norm{Q - \tilde{Q}}$ and $\frac{\norm{R - \tilde{R}}}{\norm{R}}$, that are both much larger, as reported in \autoref{fig:QR-forward}. The forward error on $Q$ is slightly worse than on $R$ due to its orthogonality property that needs to be maintained in the factorization, fixing $\kappa(\hat{X})$. However, we can see a generally decreasing trend of the forward error with respect to the condition number of the matrix $\hat{X}$. + +\begin{figure}[H] + + \centering + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/QR-lambda-error.png} % chktex 8 + \caption{\textit{QR decomposition errors and backward stability for different} $\lambda$}\label{fig:QR-error-lambda} + + \end{subfigure} + \hspace{0cm} + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/QR-forward_error.png} % chktex 8 + \caption{\textit{QR factorization forward stability on Q and R for different} $\lambda$}\label{fig:QR-forward} + \end{subfigure} + + \caption{\textit{Errors and scalability of the QR decomposition for different values of} $\lambda$}\label{fig:qrtests} +\end{figure} + + +\vspace{1em}%% insert computations (square matrices) + +% The least squares problem solved by QR factorization is also stable and we expect that the relative error $\frac{\norm{w - w^*}}{\norm{w^*}} = O(\kappa(\hat{X}) u)$ where $w$ is the solution found by the algorithm, $w^*$ is the optimal solution and $\kappa(\hat{X})$ is the condition number of the matrix $\hat{X}$. + +% \textbf{HERE IS} $O(u)$ \textbf{AND NOT} $O(\kappa(\hat{X}u))$ + +% \vspace{1em}%% insert computations + +\newpage +\section{L-BFGS} +For the first experiment regarding this algorithm we compute the relative gap, the residual and the number of iterations employed by the algorithms to converge. The relative gap is defined as +\[ +\frac{\norm{w- w^*}}{\norm{w^*}} +\] +where $w$ is the solution found by our algorithm and $w^*$ is Julia's \textit{ground truth} coming from its standard linear system solver.\\ +The residual, instead, is defined as +\[ +\frac{\norm{\hat{X}w - \hat{y}}}{\norm{\hat{y}}} +\] +The results are shown in \autoref{fig:gradnorm-res-rel}, satisfying constraints we imposed on $\kappa(\hat{X})$. It is evident from the plots that the convergence of the method is linear and that it is able to compute a relatively good solution in a small number of iterations. + +\begin{figure}[H] + + \centering + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/LBFGS-iterations-gradient-ill.png} % chktex 8 + \caption{\textit{Ill-conditioned matrix}}\label{fig:gradnorm-res-rel-ill} + \end{subfigure} + \hspace{0cm} + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/LBFGS-iterations-gradient-well.png} % chktex 8 + \caption{\textit{Well-conditioned matrix}}\label{fig:gradnorm-res-rel-wellll} + \end{subfigure} + + \caption{$\norm{\nabla f}$\textit{, Residual, Relative Error of L-BFGS execution on ill and well-conditioned matrices}}\label{fig:gradnorm-res-rel} +\end{figure} + +The other test we propose regards checking the convergence of the method, when using different line search algorithms. We checked how the gradient norm changes when using Exact Line Search and Armijo-Wolfe Line Search only on the well-conditioned matrix. + +\begin{figure}[htbp] + \centering + \includegraphics[width=0.75\linewidth]{(4) - experiments/images/LBFGS-LS-gradient-comparison.png} % chktex 8 + \caption{\textit{Line Search algorithms comparison}}\label{fig:LS-comparison} +\end{figure} + +From \autoref{fig:LS-comparison} we can notice that the exact line search behaves better than the inexact line search because of the nature of the function we are optimizing. AWLS computes a step size which may lead to instability, but does converge. + +\section{Comparison between QR and L-BFGS} +The tests have been performed by fixing one between $m = 200$ and $n = 50$ and varying the other dimension from an initial value of $500$ to a value of $5500$, at intervals of $500$. The results of fixing $m$ and varying $n$ be summarized in \autoref{fig:QRvsLBFGS-time-comparison-n}, which shows a linear growth of running time with increasing $n$ for the QR decomposition and a better performance for L-BFGS, in both the ill and well-conditioned case. +The allocated memory is consistent and on the same trend as the running time as expected. + +\begin{figure}[H] + + \centering + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/QRvsLBFGS-scalability-time-illcond-n.png} % chktex 8 + \caption{\textit{Ill-conditioned matrix}}\label{fig:time-comparison-illcond-n} + \end{subfigure} + \hspace{0cm} + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-n.png} % chktex 8 + \caption{\textit{Well-conditioned matrix}}\label{fig:time-comparison-wellcond-n} + \end{subfigure} + \caption{\textit{Time and Memory scalability comparison of QR and L-BFGS on ill and well-conditioned matrices, varying \textbf{n}}}\label{fig:QRvsLBFGS-time-comparison-n} +\end{figure} + +Instead, if we fix $n$ and let $m$ vary, we get the following curves as shown in \autoref{fig:QRvsLBFGS-time-comparison-m}. Both the running time and the allocated memory of QR grows more or less quadratically with the number of columns, confirming what the theory suggests. + +\begin{figure}[H] + + \centering + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/QRvsLBFGS-scalability-time-illcond-m.png} % chktex 8 + \caption{\textit{Ill-conditioned matrix}}\label{fig:time-comparison-illcond-m} + \end{subfigure} + \hspace{0cm} + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-m.png} % chktex 8 + \caption{\textit{Well-conditioned matrix}}\label{fig:time-comparison-wellcond-m} + \end{subfigure} + \caption{\textit{Time and Memory scalability comparison of QR and L-BFGS on ill and well-conditioned matrices, varying \textbf{m}}}\label{fig:QRvsLBFGS-time-comparison-m} +\end{figure} + + For QR the allocated memory is in the order of MiB even in the worst case while L-BFGS allocates much less memory, in the order of KiB. + +The conditioning of the matrix has no impact on the time taken to compute a solution for the two algorithms compared, but rather has an impact for L-BFGS in the quality of the solution when dealing with a very flat function (small $\lambda)$. When the function is flat it means that its curvature is low and the gradients change slowly, so the algorithm struggles to rapidly descent towards the minimum with a reasonable relative error. + +\section{Other Experiments} +\label{sec:other_experiments} +\subsection{The Effect of the Memory Size} +It is interesting to check the behaviour of L-BFGS when changing the memory size. We compare the relative error decrease at each iteration with a memory size that varies from $1$ to $11$, as shown in \autoref{fig:error-memory-size}: +\begin{figure}[H] + + \centering + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/LBFGS-iterations-memory-ill.png} % chktex 8 + \caption{\textit{Ill-conditioned matrix}}\label{fig:error-memory-size-illcond} + \end{subfigure} + \hspace{0cm} + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/LBFGS-iterations-memory-well.png} % chktex 8 + \caption{\textit{Well-conditioned matrix}}\label{fig:error-memory-size-wellcond} + \end{subfigure} + \caption{\textit{The effect of the memory size}}\label{fig:error-memory-size} + +\end{figure} + +In accordance with the suggestions provided by \cite{Numerical-Optimization-2006}, the memory size $k$ should be chosen such that $3 \leq k \leq 20$, as it is empirically a good trade-off between number of function evaluations and number of additional operations required to reconstruct the hessian with the two loop formula (\autoref{algo: L-BFGS Two-Loop Recursion}). However, since the function that has to be optimized is quadratic, the algorithm is fast at finding the optimal solution more or less independently of the memory size, but depends still on the curvature $\kappa(\hat{X})$ for the convergence. +When the memory size is $1$, the algorithm is a normal gradient descent and still reaches similar convergence rate with respect to higher memory sizes. For higher memory sizes the convergence rate is almost indistinguishable for the well-conditioned case (\autoref{fig:error-memory-size-wellcond}). However, for the ill-conditioned case, shown in \autoref{fig:error-memory-size-illcond}, the algorithm can still converge in $16$ iterations without depending on the memory size $k$, but the relative error is constant in each different setting. This is the consequence of the fact that the algorithm was terminating in a flatter region in which the curvature is so low that satisfies the stopping criterion imposed on the gradient, but with a bad approximation of the optimum. + +\subsection{A Comparison of Quasi-Newton Methods} +To further check the behaviour of our implementation of L-BFGS, we implemented and tested a version of BFGS. In the beginning of this section we provide two additional tests, performed only on well-conditioned matrices and in which we compare the two solvers together. As far as the setup is concerned, we stick to the default setup stated in \autoref{ch: experiments} and for BFGS we set the tolerance to the same as L-BFGS. The first test is shown in \autoref{fig:BFGSvsLBFGS-rate} and shows how for the least squares problem the two algorithms are almost identical in terms of convergence rate. In the plot we have the relative error, residual and gradient norm to be almost equal between the two algorithms. To understand deeply and check the differences in the implementation, we also checked the time and memory scalability. +It is not surprising that, as \autoref{fig:BFGSvsLBFGS-scalability} suggests, BFGS is much slower in finding the optimum than its limited version, even for this small optimization problem. +This aspect reflects the theory and confirms that this method is more expensive in terms of time and memory with respect to L-BFGS. + +\begin{figure}[H] + + \centering + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/BFGS-LBFGS-gradient-comparison.png} % chktex 8 + \caption{\textit{Convergence rate}}\label{fig:BFGSvsLBFGS-rate} + \end{subfigure} + \hspace{0cm} + \begin{subfigure}{0.46\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/BFGSvsLBFGS-time-m.png} % chktex 8 + \caption{\textit{Time and memory scalability}}\label{fig:BFGSvsLBFGS-scalability} + \end{subfigure} + \caption{\textit{BFGS vs L-BFGS}}\label{fig:BFGSvsLBFGS-comparison} + +\end{figure} + +To enhance and point out better the effectiveness of the implementation, we analyze other Quasi-Newton methods. We provide a final comparison between a relevant subset, in particular the final comparison consists of confronting L-BFGS, BFGS, DFP and SR1. Both DFP and BFGS are tested in its variants with the Dogleg alternative to line search as well \cite{Dogleg}. +These algorithms have been implemented and finally optimized in terms of efficiency in memory allocations, since they are prone to huge memory allocation. + +The plot in \autoref{fig:Quasi-newton-time} shows the running time of the algorithms on growing size well-conditioned matrices. In \autoref{fig:Quasi-newton-dogleg-time}, we can see that combining the update formula (BFGS and DFP) with the Dogleg (trust region) brings a lot of inefficiency in finding the minimum of that region since with line search an exact solution is used. +In the plot in \autoref{fig:Quasi-newton-no-dogleg-time} we can have a clearer visualization of the difference in efficiency between the methods, since the running time when using the Dogleg is much higher and worsens the plot. In particular, the results stick with the theory from which we expect exact line search to be better than dogleg method for finding appropriate steps; we expect also for L-BFGS to be the fastest method, followed by BFGS and SR1 that are almost equally efficient on average. The slowest is DFP that is more than twice slower than the two previously mentioned algorithms. + + +\begin{figure}[H] + + \centering + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond-Dogleg.png} + \caption{\textit{With Dogleg}} + \label{fig:Quasi-newton-dogleg-time} + \end{subfigure} + \hspace{0cm} + \begin{subfigure}{0.48\linewidth} + \centering + \includegraphics[width=\linewidth]{(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond.png} + \caption{\textit{Without Dogleg}} + \label{fig:Quasi-newton-no-dogleg-time} + \end{subfigure} + \caption{\textit{Quasi-Newton methods running time comparison}}\label{fig:Quasi-newton-time} + +\end{figure} + +The last test regards the memory allocation provided by the algorithms. This test is the equivalent of the time scalability, but the metric is the number of allocated bytes on average by the algorithms. +Our implementation has been optimized as much as possible, for instance by using Julia's in-place operators in order to minimize the number of allocations. +The last plot we display, in \autoref{fig:Quasi newton comparison memory}, shows the trend of increasing allocated bytes by the algorithms. +Methods that converge more slowly or use more memory per iteration, by using more complex update rules, perform worse. + +\begin{figure}[htbp] + \centering + \includegraphics[width=0.65\linewidth]{(4) - experiments/images/Quasi-Newton-Comparison-memory-wellcond.png} + \caption{\textit{Memory allocation of the different Quasi-Newton methods}} + \label{fig:Quasi newton comparison memory} +\end{figure} \ No newline at end of file diff --git a/Report/(4) - experiments/images/BFGS-LBFGS-gradient-comparison.png b/Report/(4) - experiments/images/BFGS-LBFGS-gradient-comparison.png new file mode 100644 index 0000000..824a56e Binary files /dev/null and b/Report/(4) - experiments/images/BFGS-LBFGS-gradient-comparison.png differ diff --git a/Report/(4) - experiments/images/BFGSvsLBFGS-time-m.png b/Report/(4) - experiments/images/BFGSvsLBFGS-time-m.png new file mode 100644 index 0000000..ad35234 Binary files /dev/null and b/Report/(4) - experiments/images/BFGSvsLBFGS-time-m.png differ diff --git a/Report/(4) - experiments/images/LBFGS-LS-gradient-comparison.png b/Report/(4) - experiments/images/LBFGS-LS-gradient-comparison.png new file mode 100644 index 0000000..53365d4 Binary files /dev/null and b/Report/(4) - experiments/images/LBFGS-LS-gradient-comparison.png differ diff --git a/Report/(4) - experiments/images/LBFGS-iterations-gradient-ill.png b/Report/(4) - experiments/images/LBFGS-iterations-gradient-ill.png new file mode 100644 index 0000000..b0491e4 Binary files /dev/null and b/Report/(4) - experiments/images/LBFGS-iterations-gradient-ill.png differ diff --git a/Report/(4) - experiments/images/LBFGS-iterations-gradient-well.png b/Report/(4) - experiments/images/LBFGS-iterations-gradient-well.png new file mode 100644 index 0000000..86e0038 Binary files /dev/null and b/Report/(4) - experiments/images/LBFGS-iterations-gradient-well.png differ diff --git a/Report/(4) - experiments/images/LBFGS-iterations-memory-ill.png b/Report/(4) - experiments/images/LBFGS-iterations-memory-ill.png new file mode 100644 index 0000000..b3e2475 Binary files /dev/null and b/Report/(4) - experiments/images/LBFGS-iterations-memory-ill.png differ diff --git a/Report/(4) - experiments/images/LBFGS-iterations-memory-well.png b/Report/(4) - experiments/images/LBFGS-iterations-memory-well.png new file mode 100644 index 0000000..2b7387c Binary files /dev/null and b/Report/(4) - experiments/images/LBFGS-iterations-memory-well.png differ diff --git a/Report/(4) - experiments/images/QR-forward_error.png b/Report/(4) - experiments/images/QR-forward_error.png new file mode 100644 index 0000000..d9289e4 Binary files /dev/null and b/Report/(4) - experiments/images/QR-forward_error.png differ diff --git a/Report/(4) - experiments/images/QR-lambda-error.png b/Report/(4) - experiments/images/QR-lambda-error.png new file mode 100644 index 0000000..954c7b6 Binary files /dev/null and b/Report/(4) - experiments/images/QR-lambda-error.png differ diff --git a/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-illcond-m.png b/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-illcond-m.png new file mode 100644 index 0000000..66fbdcd Binary files /dev/null and b/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-illcond-m.png differ diff --git a/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-illcond-n.png b/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-illcond-n.png new file mode 100644 index 0000000..f35f92e Binary files /dev/null and b/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-illcond-n.png differ diff --git a/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-m.png b/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-m.png new file mode 100644 index 0000000..74dc3c1 Binary files /dev/null and b/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-m.png differ diff --git a/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-n.png b/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-n.png new file mode 100644 index 0000000..277a977 Binary files /dev/null and b/Report/(4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-n.png differ diff --git a/Report/(4) - experiments/images/Quasi-Newton-Comparison-memory-wellcond.png b/Report/(4) - experiments/images/Quasi-Newton-Comparison-memory-wellcond.png new file mode 100644 index 0000000..bf577a7 Binary files /dev/null and b/Report/(4) - experiments/images/Quasi-Newton-Comparison-memory-wellcond.png differ diff --git a/Report/(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond-Dogleg.png b/Report/(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond-Dogleg.png new file mode 100644 index 0000000..d6953c8 Binary files /dev/null and b/Report/(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond-Dogleg.png differ diff --git a/Report/(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond.png b/Report/(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond.png new file mode 100644 index 0000000..9fd461a Binary files /dev/null and b/Report/(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond.png differ diff --git a/Report/(6) - conclusion/conclusion.aux b/Report/(6) - conclusion/conclusion.aux new file mode 100644 index 0000000..d9a0be7 --- /dev/null +++ b/Report/(6) - conclusion/conclusion.aux @@ -0,0 +1,168 @@ +\relax +\providecommand\hyper@newdestlabel[2]{} +\@writefile{toc}{\contentsline {chapter}{\numberline {5}Concluding Remarks}{21}{chapter.5}\protected@file@percent } +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loa}{\addvspace {10\p@ }} +\newlabel{ch: conclusion}{{5}{21}{Concluding Remarks}{chapter.5}{}} +\@setckpt{(6) - conclusion/conclusion}{ +\setcounter{page}{22} +\setcounter{equation}{0} +\setcounter{enumi}{3} +\setcounter{enumii}{0} +\setcounter{enumiii}{0} +\setcounter{enumiv}{0} +\setcounter{footnote}{0} +\setcounter{mpfootnote}{0} +\setcounter{part}{0} +\setcounter{chapter}{5} +\setcounter{section}{0} +\setcounter{subsection}{0} +\setcounter{subsubsection}{0} +\setcounter{paragraph}{0} +\setcounter{subparagraph}{0} +\setcounter{figure}{0} +\setcounter{table}{0} +\setcounter{tabx@nest}{0} +\setcounter{listtotal}{0} +\setcounter{listcount}{0} +\setcounter{liststart}{0} +\setcounter{liststop}{0} +\setcounter{citecount}{0} +\setcounter{citetotal}{0} +\setcounter{multicitecount}{0} +\setcounter{multicitetotal}{0} +\setcounter{instcount}{5} +\setcounter{maxnames}{3} +\setcounter{minnames}{3} +\setcounter{maxitems}{3} +\setcounter{minitems}{1} +\setcounter{citecounter}{0} +\setcounter{maxcitecounter}{0} +\setcounter{savedcitecounter}{0} +\setcounter{uniquelist}{0} +\setcounter{uniquename}{0} +\setcounter{refsection}{0} +\setcounter{refsegment}{0} +\setcounter{maxextratitle}{0} +\setcounter{maxextratitleyear}{0} +\setcounter{maxextraname}{0} +\setcounter{maxextradate}{0} +\setcounter{maxextraalpha}{0} +\setcounter{abbrvpenalty}{50} +\setcounter{highnamepenalty}{50} +\setcounter{lownamepenalty}{25} +\setcounter{maxparens}{3} +\setcounter{parenlevel}{0} +\setcounter{blx@maxsection}{0} +\setcounter{mincomprange}{10} +\setcounter{maxcomprange}{100000} +\setcounter{mincompwidth}{1} +\setcounter{afterword}{0} +\setcounter{savedafterword}{0} +\setcounter{annotator}{0} +\setcounter{savedannotator}{0} +\setcounter{author}{0} +\setcounter{savedauthor}{0} +\setcounter{bookauthor}{0} +\setcounter{savedbookauthor}{0} +\setcounter{commentator}{0} +\setcounter{savedcommentator}{0} +\setcounter{editor}{0} +\setcounter{savededitor}{0} +\setcounter{editora}{0} +\setcounter{savededitora}{0} +\setcounter{editorb}{0} +\setcounter{savededitorb}{0} +\setcounter{editorc}{0} +\setcounter{savededitorc}{0} +\setcounter{foreword}{0} +\setcounter{savedforeword}{0} +\setcounter{holder}{0} +\setcounter{savedholder}{0} +\setcounter{introduction}{0} +\setcounter{savedintroduction}{0} +\setcounter{namea}{0} +\setcounter{savednamea}{0} +\setcounter{nameb}{0} +\setcounter{savednameb}{0} +\setcounter{namec}{0} +\setcounter{savednamec}{0} +\setcounter{translator}{0} +\setcounter{savedtranslator}{0} +\setcounter{shortauthor}{0} +\setcounter{savedshortauthor}{0} +\setcounter{shorteditor}{0} +\setcounter{savedshorteditor}{0} +\setcounter{labelname}{0} +\setcounter{savedlabelname}{0} +\setcounter{institution}{0} +\setcounter{savedinstitution}{0} +\setcounter{lista}{0} +\setcounter{savedlista}{0} +\setcounter{listb}{0} +\setcounter{savedlistb}{0} +\setcounter{listc}{0} +\setcounter{savedlistc}{0} +\setcounter{listd}{0} +\setcounter{savedlistd}{0} +\setcounter{liste}{0} +\setcounter{savedliste}{0} +\setcounter{listf}{0} +\setcounter{savedlistf}{0} +\setcounter{location}{0} +\setcounter{savedlocation}{0} +\setcounter{organization}{0} +\setcounter{savedorganization}{0} +\setcounter{origlocation}{0} +\setcounter{savedoriglocation}{0} +\setcounter{origpublisher}{0} +\setcounter{savedorigpublisher}{0} +\setcounter{publisher}{0} +\setcounter{savedpublisher}{0} +\setcounter{language}{0} +\setcounter{savedlanguage}{0} +\setcounter{origlanguage}{0} +\setcounter{savedoriglanguage}{0} +\setcounter{pageref}{0} +\setcounter{savedpageref}{0} +\setcounter{textcitecount}{0} +\setcounter{textcitetotal}{0} +\setcounter{textcitemaxnames}{0} +\setcounter{biburlbigbreakpenalty}{100} +\setcounter{biburlbreakpenalty}{200} +\setcounter{biburlnumpenalty}{0} +\setcounter{biburlucpenalty}{0} +\setcounter{biburllcpenalty}{0} +\setcounter{smartand}{1} +\setcounter{bbx:relatedcount}{0} +\setcounter{bbx:relatedtotal}{0} +\setcounter{cbx@tempcnta}{0} +\setcounter{cbx@tempcntb}{2} +\setcounter{cbx@tempcntc}{0} +\setcounter{cbx@tempcntd}{-1} +\setcounter{float@type}{4} +\setcounter{parentequation}{0} +\setcounter{ALG@line}{0} +\setcounter{ALG@rem}{0} +\setcounter{ALG@nested}{0} +\setcounter{ALG@Lnr}{2} +\setcounter{ALG@blocknr}{10} +\setcounter{ALG@storecount}{0} +\setcounter{ALG@tmpcounter}{0} +\setcounter{thmt@dummyctr}{2} +\setcounter{nlinenum}{0} +\setcounter{caption@flags}{2} +\setcounter{continuedfloat}{0} +\setcounter{subfigure}{0} +\setcounter{subtable}{0} +\setcounter{section@level}{0} +\setcounter{Item}{3} +\setcounter{Hfootnote}{0} +\setcounter{bookmark@seq@number}{0} +\setcounter{AlgoLine}{11} +\setcounter{algocfline}{4} +\setcounter{algocfproc}{4} +\setcounter{algocf}{4} +\setcounter{mlemma}{0} +} diff --git a/Report/(6) - conclusion/conclusion.tex b/Report/(6) - conclusion/conclusion.tex new file mode 100644 index 0000000..f24bb0a --- /dev/null +++ b/Report/(6) - conclusion/conclusion.tex @@ -0,0 +1,5 @@ +\chapter{Concluding Remarks}\label{ch: conclusion} + +An implementation of the thin-QR factorization and limited memory BFGS has been presented, in particular with exact line search in order to solve more efficiently the least squares problem. Convergence for both methods have been proven and tested. +An implementation of BFGS and DFP, with both exact line search and trust region method using the dogleg method, and SR1 have been implemented and tested. +From the experiments it is pretty clear that L-BFGS is better than all other Quasi-Newton methods for solving the least squares problem, even when using a small parameter for the memory. Instead the QR method performs better when solving for ill-conditioned matrices, but the memory usage is higher. \ No newline at end of file diff --git a/Report/(7) - proofs/proofs.aux b/Report/(7) - proofs/proofs.aux new file mode 100644 index 0000000..e4e883a --- /dev/null +++ b/Report/(7) - proofs/proofs.aux @@ -0,0 +1,179 @@ +\relax +\providecommand\hyper@newdestlabel[2]{} +\@writefile{toc}{\contentsline {chapter}{\numberline {6}Proofs}{22}{chapter.6}\protected@file@percent } +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loa}{\addvspace {10\p@ }} +\newlabel{ch: proofs}{{6}{22}{Proofs}{chapter.6}{}} +\@writefile{loe}{\addvspace {10\p@ }} +\@writefile{loe}{\contentsline {mlemma}{\ifthmt@listswap Lemma~1\else \numberline {1}Lemma\fi }{22}{mlemma.1}\protected@file@percent } +\newlabel{proofs: fullcolumn}{{1}{22}{}{mlemma.1}{}} +\newlabel{proof:triangularsystem}{{1}{22}{}{mlemma.1}{}} +\@writefile{loe}{\contentsline {mproof}{\ifthmt@listswap \else \numberline {\let \autodot \@empty }\fi Proof}{22}{thmt@dummyctr.dummy.4}\protected@file@percent } +\@writefile{loe}{\contentsline {mlemma}{\ifthmt@listswap Lemma~2\else \numberline {2}Lemma\fi }{22}{mlemma.2}\protected@file@percent } +\newlabel{proof:eigenvalues_translation}{{2}{22}{}{mlemma.2}{}} +\@writefile{loe}{\contentsline {mproof}{\ifthmt@listswap \else \numberline {\let \autodot \@empty }\fi Proof}{22}{thmt@dummyctr.dummy.6}\protected@file@percent } +\@writefile{loe}{\contentsline {mlemma}{\ifthmt@listswap Lemma~3\else \numberline {3}Lemma\fi }{22}{mlemma.3}\protected@file@percent } +\newlabel{proofs: eigenvalues}{{3}{22}{}{mlemma.3}{}} +\@writefile{loe}{\contentsline {mproof}{\ifthmt@listswap \else \numberline {\let \autodot \@empty }\fi Proof}{22}{thmt@dummyctr.dummy.8}\protected@file@percent } +\@setckpt{(7) - proofs/proofs}{ +\setcounter{page}{23} +\setcounter{equation}{0} +\setcounter{enumi}{3} +\setcounter{enumii}{0} +\setcounter{enumiii}{0} +\setcounter{enumiv}{0} +\setcounter{footnote}{0} +\setcounter{mpfootnote}{0} +\setcounter{part}{0} +\setcounter{chapter}{6} +\setcounter{section}{0} +\setcounter{subsection}{0} +\setcounter{subsubsection}{0} +\setcounter{paragraph}{0} +\setcounter{subparagraph}{0} +\setcounter{figure}{0} +\setcounter{table}{0} +\setcounter{tabx@nest}{0} +\setcounter{listtotal}{0} +\setcounter{listcount}{0} +\setcounter{liststart}{0} +\setcounter{liststop}{0} +\setcounter{citecount}{0} +\setcounter{citetotal}{0} +\setcounter{multicitecount}{0} +\setcounter{multicitetotal}{0} +\setcounter{instcount}{5} +\setcounter{maxnames}{3} +\setcounter{minnames}{3} +\setcounter{maxitems}{3} +\setcounter{minitems}{1} +\setcounter{citecounter}{0} +\setcounter{maxcitecounter}{0} +\setcounter{savedcitecounter}{0} +\setcounter{uniquelist}{0} +\setcounter{uniquename}{0} +\setcounter{refsection}{0} +\setcounter{refsegment}{0} +\setcounter{maxextratitle}{0} +\setcounter{maxextratitleyear}{0} +\setcounter{maxextraname}{0} +\setcounter{maxextradate}{0} +\setcounter{maxextraalpha}{0} +\setcounter{abbrvpenalty}{50} +\setcounter{highnamepenalty}{50} +\setcounter{lownamepenalty}{25} +\setcounter{maxparens}{3} +\setcounter{parenlevel}{0} +\setcounter{blx@maxsection}{0} +\setcounter{mincomprange}{10} +\setcounter{maxcomprange}{100000} +\setcounter{mincompwidth}{1} +\setcounter{afterword}{0} +\setcounter{savedafterword}{0} +\setcounter{annotator}{0} +\setcounter{savedannotator}{0} +\setcounter{author}{0} +\setcounter{savedauthor}{0} +\setcounter{bookauthor}{0} +\setcounter{savedbookauthor}{0} +\setcounter{commentator}{0} +\setcounter{savedcommentator}{0} +\setcounter{editor}{0} +\setcounter{savededitor}{0} +\setcounter{editora}{0} +\setcounter{savededitora}{0} +\setcounter{editorb}{0} +\setcounter{savededitorb}{0} +\setcounter{editorc}{0} +\setcounter{savededitorc}{0} +\setcounter{foreword}{0} +\setcounter{savedforeword}{0} +\setcounter{holder}{0} +\setcounter{savedholder}{0} +\setcounter{introduction}{0} +\setcounter{savedintroduction}{0} +\setcounter{namea}{0} +\setcounter{savednamea}{0} +\setcounter{nameb}{0} +\setcounter{savednameb}{0} +\setcounter{namec}{0} +\setcounter{savednamec}{0} +\setcounter{translator}{0} +\setcounter{savedtranslator}{0} +\setcounter{shortauthor}{0} +\setcounter{savedshortauthor}{0} +\setcounter{shorteditor}{0} +\setcounter{savedshorteditor}{0} +\setcounter{labelname}{0} +\setcounter{savedlabelname}{0} +\setcounter{institution}{0} +\setcounter{savedinstitution}{0} +\setcounter{lista}{0} +\setcounter{savedlista}{0} +\setcounter{listb}{0} +\setcounter{savedlistb}{0} +\setcounter{listc}{0} +\setcounter{savedlistc}{0} +\setcounter{listd}{0} +\setcounter{savedlistd}{0} +\setcounter{liste}{0} +\setcounter{savedliste}{0} +\setcounter{listf}{0} +\setcounter{savedlistf}{0} +\setcounter{location}{0} +\setcounter{savedlocation}{0} +\setcounter{organization}{0} +\setcounter{savedorganization}{0} +\setcounter{origlocation}{0} +\setcounter{savedoriglocation}{0} +\setcounter{origpublisher}{0} +\setcounter{savedorigpublisher}{0} +\setcounter{publisher}{0} +\setcounter{savedpublisher}{0} +\setcounter{language}{0} +\setcounter{savedlanguage}{0} +\setcounter{origlanguage}{0} +\setcounter{savedoriglanguage}{0} +\setcounter{pageref}{0} +\setcounter{savedpageref}{0} +\setcounter{textcitecount}{0} +\setcounter{textcitetotal}{0} +\setcounter{textcitemaxnames}{0} +\setcounter{biburlbigbreakpenalty}{100} +\setcounter{biburlbreakpenalty}{200} +\setcounter{biburlnumpenalty}{0} +\setcounter{biburlucpenalty}{0} +\setcounter{biburllcpenalty}{0} +\setcounter{smartand}{1} +\setcounter{bbx:relatedcount}{0} +\setcounter{bbx:relatedtotal}{0} +\setcounter{cbx@tempcnta}{0} +\setcounter{cbx@tempcntb}{2} +\setcounter{cbx@tempcntc}{0} +\setcounter{cbx@tempcntd}{-1} +\setcounter{float@type}{4} +\setcounter{parentequation}{0} +\setcounter{ALG@line}{0} +\setcounter{ALG@rem}{0} +\setcounter{ALG@nested}{0} +\setcounter{ALG@Lnr}{2} +\setcounter{ALG@blocknr}{10} +\setcounter{ALG@storecount}{0} +\setcounter{ALG@tmpcounter}{0} +\setcounter{thmt@dummyctr}{8} +\setcounter{nlinenum}{0} +\setcounter{caption@flags}{2} +\setcounter{continuedfloat}{0} +\setcounter{subfigure}{0} +\setcounter{subtable}{0} +\setcounter{section@level}{0} +\setcounter{Item}{3} +\setcounter{Hfootnote}{0} +\setcounter{bookmark@seq@number}{0} +\setcounter{AlgoLine}{11} +\setcounter{algocfline}{4} +\setcounter{algocfproc}{4} +\setcounter{algocf}{4} +\setcounter{mlemma}{3} +} diff --git a/Report/(7) - proofs/proofs.tex b/Report/(7) - proofs/proofs.tex new file mode 100644 index 0000000..1582411 --- /dev/null +++ b/Report/(7) - proofs/proofs.tex @@ -0,0 +1,52 @@ +\chapter{Proofs}\label{ch: proofs} + +\begin{mlemma}\label{proofs: fullcolumn}\label{proof:triangularsystem} + $\hat{X} \text{ has full column rank } \iff \hat{X}^T\hat{X} \succ 0$ +\end{mlemma} + +\begin{mproof} +To show that $\hat{X}^T\hat{X} = XX^T + \lambda^2 I_m \succ 0$, $\lambda > 0$, we can consider the quadratic form $x^T (X^{T}X + \lambda^2 I_m) x$. Let $B = XX^T \succeq 0$. +\begin{align*} + x^T (B + \lambda^2 I_m) x &= x^{T}Bx + \lambda^2 x^{T}I_{m}x \\ + &= x^{T}Bx + \lambda^2 \norm{x}^2 +\end{align*} + +Since $B$ is positive semidefinite, we have $x^{T}Bx \geq 0\ \ \forall x \in \mathbb{R}^m$. Additionally, $\lambda^2 \|x\|^2 > 0\ \ \forall x \neq 0$. Therefore, $x^T (B + \lambda^2 I_m) x > 0$ for all non-zero vectors $x$, meaning that $\hat{X}^T\hat{X} \succ 0$. + +\end{mproof} + +\begin{mlemma}\label{proof:eigenvalues_translation} + $\alpha \in Sp(A) \iff (\alpha + \lambda) \in Sp(A + \lambda I)$ +\end{mlemma} + +\begin{mproof} + $Av = \alpha v \iff (A + \lambda I)v = Av + \lambda v = \alpha v + \lambda v = (\alpha + \lambda)v$ +\end{mproof} + +\begin{mlemma}\label{proofs: eigenvalues} + The singular values of the matrix $XX^T, X \in \mathbb{R}^{m \times n}$\newline are $\{\sigma_1^2 \dots \sigma_n^2, 0 \dots 0\}$, with $\sigma_1 \dots \sigma_n$ being the singular values of $X$. +\end{mlemma} + +\begin{mproof} + Consider the Singular Value Decomposition of the rank $n$ matrix $X$ + \[ + X = U \Sigma V^T + \] + $\Sigma = \diag(\sigma_1, \dots, \sigma_n) \in \mathbb{R}^{m \times n} + $ + Then + \[ + XX^T = U \Sigma V^T V \Sigma^T U^T = U \Sigma \Sigma^T U + \] + with + \[ + \Sigma \Sigma^T = \diag(\sigma_1^2, \dots, \sigma_n^2, 0, \dots, 0) \in \mathbb{R}^{m \times m} + \] + Hence, $XX^T$ has exactly $m$ singular values of which $m-n$ are zeros. +\end{mproof} + +%%% Local Variables: +%%% mode: latex +%%% TeX-master: "../main" +%%% TeX-command-extra-options: "-shell-escape" +%%% End: diff --git a/Report/auto/main.el b/Report/auto/main.el new file mode 100644 index 0000000..cc7cef3 --- /dev/null +++ b/Report/auto/main.el @@ -0,0 +1,102 @@ +;; -*- lexical-binding: t; -*- + +(TeX-add-style-hook + "main" + (lambda () + (setq TeX-command-extra-options + "-shell-escape") + (TeX-add-to-alist 'LaTeX-provided-class-options + '(("report" "12pt"))) + (TeX-add-to-alist 'LaTeX-provided-package-options + '(("geometry" "headheight=20pt" "centering") ("inputenc" "utf8") ("babel" "english") ("scrlayer-scrpage" "") ("biblatex" "backend=biber" "style=ieee" "sorting=ynt") ("import" "") ("graphicx" "") ("float" "") ("amssymb" "") ("amsmath" "") ("tabularray" "") ("booktabs" "") ("mathrsfs" "") ("mathtools" "") ("algpseudocode" "") ("amsthm" "") ("thmtools" "") ("pgfplots" "") ("tikz" "") ("enumitem" "") ("color" "") ("xcolor" "") ("colortbl" "") ("makecell" "") ("cancel" "") ("pgfornament" "") ("multicol" "") ("subcaption" "") ("bold-extra" "") ("csquotes" "") ("hyperref" "hidelinks") ("bookmark" "") ("algorithm2e" "ruled" "vlined" "linesnumbered") ("titlesec" "") ("setspace" ""))) + (add-to-list 'LaTeX-verbatim-environments-local "minted") + (add-to-list 'LaTeX-verbatim-macros-with-braces-local "href") + (add-to-list 'LaTeX-verbatim-macros-with-braces-local "hyperimage") + (add-to-list 'LaTeX-verbatim-macros-with-braces-local "hyperbaseurl") + (add-to-list 'LaTeX-verbatim-macros-with-braces-local "nolinkurl") + (add-to-list 'LaTeX-verbatim-macros-with-braces-local "url") + (add-to-list 'LaTeX-verbatim-macros-with-braces-local "path") + (add-to-list 'LaTeX-verbatim-macros-with-delims-local "path") + (TeX-run-style-hooks + "latex2e" + "(1) - introduction/introduction" + "(2) - problem definition/problem definition" + "(3) - algorithms/algorithms" + "(4) - experiments/experiments" + "(6) - conclusion/conclusion" + "(7) - proofs/proofs" + "report" + "rep12" + "geometry" + "inputenc" + "babel" + "scrlayer-scrpage" + "biblatex" + "import" + "graphicx" + "float" + "amssymb" + "amsmath" + "tabularray" + "booktabs" + "mathrsfs" + "mathtools" + "algpseudocode" + "amsthm" + "thmtools" + "pgfplots" + "tikz" + "enumitem" + "color" + "xcolor" + "colortbl" + "makecell" + "cancel" + "pgfornament" + "multicol" + "subcaption" + "bold-extra" + "csquotes" + "hyperref" + "bookmark" + "algorithm2e" + "titlesec" + "setspace") + (TeX-add-symbols + '("ieb" ["argument"] 1) + '("thend" 1) + '("norm" 1) + '("vec" 1) + "zerodisplayskips" + "lgt" + "gry" + "rd" + "logoA" + "smallLogoA" + "course" + "assignment" + "authors" + "emails" + "universityA" + "academicYear" + "department") + (LaTeX-add-environments + "olditemize") + (LaTeX-add-bibliographies + "document") + (LaTeX-add-thmtools-declaretheoremstyles + "steo" + "slemma" + "sdef" + "sprf") + (LaTeX-add-thmtools-declaretheorems + "mtheo" + "mlemma" + "mdef" + "mproof") + (LaTeX-add-color-definecolors + "red" + "lightgreen" + "gray")) + :latex) + diff --git a/Report/document.bib b/Report/document.bib new file mode 100644 index 0000000..cd4cd3f --- /dev/null +++ b/Report/document.bib @@ -0,0 +1,51 @@ +@Book{Numerical-Optimization-2006, + author = {Jorge Nocedal and Stephen J. Wright}, + publisher = {Springer}, + title = {Numerical Optimization}, + year = {2006}, + address = {New York, NY, USA}, + edition = {2e}, +} + +@article{convergence_lbfgs, + title = {On the limited memory BFGS method for large scale optimization}, + volume = {45}, + DOI = {10.1007/bf01589116}, + number = {1–3}, + journal = {Mathematical Programming}, + author = {Liu, Dong C. and Nocedal, Jorge}, + year = {1989}, + month = {8}, + pages = {503–528} +} + +@ARTICLE{BenchmarkTools, + author = {{Chen}, Jiahao and {Revels}, Jarrett}, + title = "{Robust benchmarking in noisy environments}", + journal = {arXiv e-prints}, + keywords = {Computer Science - Performance, 68N30, B.8.1, D.2.5}, + year = 2016, + month = 8, + eid = {arXiv:1608.04295}, + archivePrefix = {arXiv}, + eprint = {1608.04295}, + primaryClass = {cs.PF}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2016arXiv160804295C}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@INPROCEEDINGS {Dogleg, + author = {N. Ampazis and S. Spirou and S. Perantonis}, + booktitle = {Neural Networks, IEEE - INNS - ENNS International Joint Conference on}, + title = {Training Feedforward Neural Networks with the Dogleg Method and BFGS Hessian Updates}, + year = {2000}, + volume = {2}, + issn = {1098-7576}, + pages = {1138}, + abstract = {In this paper, we introduce an advanced optimization algorithm for training feedforward neural networks. The algorithm combines the BFGS Hessian update formula with a special case of trust region techniques, called the Dogleg method, as an altenative technique to line search methods. Simulations regarding classification and function approximation problems are presented which reveal a clear improvement both in convergence and success rates over standard BFGS implementations.}, + doi = {10.1109/IJCNN.2000.857827}, + url = {https://doi.ieeecomputersociety.org/10.1109/IJCNN.2000.857827}, + publisher = {IEEE Computer Society}, + address = {Los Alamitos, CA, USA}, + month = 7 +} diff --git a/Report/images/UniPi Logo.png b/Report/images/UniPi Logo.png new file mode 100644 index 0000000..b32ebf5 Binary files /dev/null and b/Report/images/UniPi Logo.png differ diff --git a/Report/main.aux b/Report/main.aux new file mode 100644 index 0000000..7ed1090 --- /dev/null +++ b/Report/main.aux @@ -0,0 +1,49 @@ +\relax +\providecommand\babel@aux[2]{} +\@nameuse{bbl@beforestart} +\abx@aux@refcontext{ynt/global//global/global/global} +\providecommand\hyper@newdestlabel[2]{} +\providecommand\HyField@AuxAddToFields[1]{} +\providecommand\HyField@AuxAddToCoFields[2]{} +\providecommand\BKM@entry[2]{} +\providecommand \oddpage@label [2]{} +\babel@aux{english}{} +\@input{(1) - introduction/introduction.aux} +\BKM@entry{id=1,dest={636861707465722E31},srcline={1}}{5C3337365C3337375C303030495C3030306E5C303030745C303030725C3030306F5C303030645C303030755C303030635C303030745C303030695C3030306F5C3030306E} +\@input{(2) - problem definition/problem definition.aux} +\BKM@entry{id=2,dest={636861707465722E32},srcline={1}}{5C3337365C3337375C303030505C303030725C3030306F5C303030625C3030306C5C303030655C3030306D5C3030305C3034305C303030445C303030655C303030665C303030695C3030306E5C303030695C303030745C303030695C3030306F5C3030306E} +\BKM@entry{id=3,dest={73656374696F6E2E322E31},srcline={6}}{5C3337365C3337375C303030515C30303052} +\BKM@entry{id=4,dest={73656374696F6E2E322E32},srcline={60}}{5C3337365C3337375C3030304C5C3030302D5C303030425C303030465C303030475C30303053} +\BKM@entry{id=5,dest={73656374696F6E2E322E33},srcline={114}}{5C3337365C3337375C303030435C3030306F5C3030306E5C303030645C303030695C303030745C303030695C3030306F5C3030306E5C303030695C3030306E5C30303067} +\@input{(3) - algorithms/algorithms.aux} +\BKM@entry{id=6,dest={636861707465722E33},srcline={2}}{5C3337365C3337375C303030415C3030306C5C303030675C3030306F5C303030725C303030695C303030745C303030685C3030306D5C30303073} +\BKM@entry{id=7,dest={73656374696F6E2E332E31},srcline={4}}{5C3337365C3337375C303030515C30303052} +\BKM@entry{id=8,dest={73656374696F6E2E332E32},srcline={87}}{5C3337365C3337375C3030304C5C3030302D5C303030425C303030465C303030475C30303053} +\abx@aux@cite{0}{Numerical-Optimization-2006} +\abx@aux@segm{0}{0}{Numerical-Optimization-2006} +\abx@aux@cite{0}{convergence_lbfgs} +\abx@aux@segm{0}{0}{convergence_lbfgs} +\@input{(4) - experiments/experiments.aux} +\BKM@entry{id=9,dest={636861707465722E34},srcline={1}}{5C3337365C3337375C303030455C303030785C303030705C303030655C303030725C303030695C3030306D5C303030655C3030306E5C303030745C30303073} +\abx@aux@cite{0}{BenchmarkTools} +\abx@aux@segm{0}{0}{BenchmarkTools} +\BKM@entry{id=10,dest={73656374696F6E2E342E31},srcline={17}}{5C3337365C3337375C303030515C30303052} +\BKM@entry{id=11,dest={73656374696F6E2E342E32},srcline={51}}{5C3337365C3337375C3030304C5C3030302D5C303030425C303030465C303030475C30303053} +\BKM@entry{id=12,dest={73656374696F6E2E342E33},srcline={91}}{5C3337365C3337375C303030435C3030306F5C3030306D5C303030705C303030615C303030725C303030695C303030735C3030306F5C3030306E5C3030305C3034305C303030625C303030655C303030745C303030775C303030655C303030655C3030306E5C3030305C3034305C303030515C303030525C3030305C3034305C303030615C3030306E5C303030645C3030305C3034305C3030304C5C3030302D5C303030425C303030465C303030475C30303053} +\BKM@entry{id=13,dest={73656374696F6E2E342E34},srcline={135}}{5C3337365C3337375C3030304F5C303030745C303030685C303030655C303030725C3030305C3034305C303030455C303030785C303030705C303030655C303030725C303030695C3030306D5C303030655C3030306E5C303030745C30303073} +\BKM@entry{id=14,dest={73756273656374696F6E2E342E342E31},srcline={137}}{5C3337365C3337375C303030545C303030685C303030655C3030305C3034305C303030455C303030665C303030665C303030655C303030635C303030745C3030305C3034305C3030306F5C303030665C3030305C3034305C303030745C303030685C303030655C3030305C3034305C3030304D5C303030655C3030306D5C3030306F5C303030725C303030795C3030305C3034305C303030535C303030695C3030307A5C30303065} +\abx@aux@cite{0}{Numerical-Optimization-2006} +\abx@aux@segm{0}{0}{Numerical-Optimization-2006} +\BKM@entry{id=15,dest={73756273656374696F6E2E342E342E32},srcline={160}}{5C3337365C3337375C303030415C3030305C3034305C303030435C3030306F5C3030306D5C303030705C303030615C303030725C303030695C303030735C3030306F5C3030306E5C3030305C3034305C3030306F5C303030665C3030305C3034305C303030515C303030755C303030615C303030735C303030695C3030302D5C3030304E5C303030655C303030775C303030745C3030306F5C3030306E5C3030305C3034305C3030304D5C303030655C303030745C303030685C3030306F5C303030645C30303073} +\abx@aux@cite{0}{Dogleg} +\abx@aux@segm{0}{0}{Dogleg} +\@input{(6) - conclusion/conclusion.aux} +\BKM@entry{id=16,dest={636861707465722E35},srcline={1}}{5C3337365C3337375C303030435C3030306F5C3030306E5C303030635C3030306C5C303030755C303030645C303030695C3030306E5C303030675C3030305C3034305C303030525C303030655C3030306D5C303030615C303030725C3030306B5C30303073} +\@input{(7) - proofs/proofs.aux} +\BKM@entry{id=17,dest={636861707465722E36},srcline={1}}{5C3337365C3337375C303030505C303030725C3030306F5C3030306F5C303030665C30303073} +\abx@aux@read@bbl@mdfivesum{51A82EAEBFB92FC698FF03FF115EBC95} +\abx@aux@defaultrefcontext{0}{convergence_lbfgs}{ynt/global//global/global/global} +\abx@aux@defaultrefcontext{0}{Dogleg}{ynt/global//global/global/global} +\abx@aux@defaultrefcontext{0}{Numerical-Optimization-2006}{ynt/global//global/global/global} +\abx@aux@defaultrefcontext{0}{BenchmarkTools}{ynt/global//global/global/global} +\gdef \@abspage@last{25} diff --git a/Report/main.bbl b/Report/main.bbl new file mode 100644 index 0000000..30ef6b9 --- /dev/null +++ b/Report/main.bbl @@ -0,0 +1,189 @@ +% $ biblatex auxiliary file $ +% $ biblatex bbl format version 3.3 $ +% Do not modify the above lines! +% +% This is an auxiliary file used by the 'biblatex' package. +% This file may safely be deleted. It will be recreated by +% biber as required. +% +\begingroup +\makeatletter +\@ifundefined{ver@biblatex.sty} + {\@latex@error + {Missing 'biblatex' package} + {The bibliography requires the 'biblatex' package.} + \aftergroup\endinput} + {} +\endgroup + + +\refsection{0} + \datalist[entry]{ynt/global//global/global/global} + \entry{convergence_lbfgs}{article}{}{} + \name{author}{2}{}{% + {{hash=3773c3d5aaf3636d59b5d646468683ae}{% + family={Liu}, + familyi={L\bibinitperiod}, + given={Dong\bibnamedelima C.}, + giveni={D\bibinitperiod\bibinitdelim C\bibinitperiod}}}% + {{hash=88a342d927bf795b0d92af8a5613da31}{% + family={Nocedal}, + familyi={N\bibinitperiod}, + given={Jorge}, + giveni={J\bibinitperiod}}}% + } + \strng{namehash}{016da99314d2fb9b907bb0a91cc52ecd} + \strng{fullhash}{016da99314d2fb9b907bb0a91cc52ecd} + \strng{fullhashraw}{016da99314d2fb9b907bb0a91cc52ecd} + \strng{bibnamehash}{016da99314d2fb9b907bb0a91cc52ecd} + \strng{authorbibnamehash}{016da99314d2fb9b907bb0a91cc52ecd} + \strng{authornamehash}{016da99314d2fb9b907bb0a91cc52ecd} + \strng{authorfullhash}{016da99314d2fb9b907bb0a91cc52ecd} + \strng{authorfullhashraw}{016da99314d2fb9b907bb0a91cc52ecd} + \field{sortinit}{1} + \field{sortinithash}{4f6aaa89bab872aa0999fec09ff8e98a} + \field{labelnamesource}{author} + \field{labeltitlesource}{title} + \field{journaltitle}{Mathematical Programming} + \field{month}{8} + \field{number}{1–3} + \field{title}{On the limited memory BFGS method for large scale optimization} + \field{volume}{45} + \field{year}{1989} + \field{pages}{503\bibrangedash 528} + \range{pages}{26} + \verb{doi} + \verb 10.1007/bf01589116 + \endverb + \endentry + \entry{Dogleg}{inproceedings}{}{} + \name{author}{3}{}{% + {{hash=f8ef8253cc84bb120a25129626f9bf75}{% + family={Ampazis}, + familyi={A\bibinitperiod}, + given={N.}, + giveni={N\bibinitperiod}}}% + {{hash=ca32e2eb0ff1d4baff79927c876e24bc}{% + family={Spirou}, + familyi={S\bibinitperiod}, + given={S.}, + giveni={S\bibinitperiod}}}% + {{hash=8857a8a8805ecdaf2863ccf5ebb56be2}{% + family={Perantonis}, + familyi={P\bibinitperiod}, + given={S.}, + giveni={S\bibinitperiod}}}% + } + \list{location}{1}{% + {Los Alamitos, CA, USA}% + } + \list{publisher}{1}{% + {IEEE Computer Society}% + } + \strng{namehash}{043d6ae890a3edd2e8dd411887d99c3c} + \strng{fullhash}{043d6ae890a3edd2e8dd411887d99c3c} + \strng{fullhashraw}{043d6ae890a3edd2e8dd411887d99c3c} + \strng{bibnamehash}{043d6ae890a3edd2e8dd411887d99c3c} + \strng{authorbibnamehash}{043d6ae890a3edd2e8dd411887d99c3c} + \strng{authornamehash}{043d6ae890a3edd2e8dd411887d99c3c} + \strng{authorfullhash}{043d6ae890a3edd2e8dd411887d99c3c} + \strng{authorfullhashraw}{043d6ae890a3edd2e8dd411887d99c3c} + \field{sortinit}{2} + \field{sortinithash}{8b555b3791beccb63322c22f3320aa9a} + \field{labelnamesource}{author} + \field{labeltitlesource}{title} + \field{abstract}{In this paper, we introduce an advanced optimization algorithm for training feedforward neural networks. The algorithm combines the BFGS Hessian update formula with a special case of trust region techniques, called the Dogleg method, as an altenative technique to line search methods. Simulations regarding classification and function approximation problems are presented which reveal a clear improvement both in convergence and success rates over standard BFGS implementations.} + \field{booktitle}{Neural Networks, IEEE - INNS - ENNS International Joint Conference on} + \field{issn}{1098-7576} + \field{month}{7} + \field{title}{Training Feedforward Neural Networks with the Dogleg Method and BFGS Hessian Updates} + \field{volume}{2} + \field{year}{2000} + \field{pages}{1138} + \range{pages}{1} + \verb{doi} + \verb 10.1109/IJCNN.2000.857827 + \endverb + \verb{urlraw} + \verb https://doi.ieeecomputersociety.org/10.1109/IJCNN.2000.857827 + \endverb + \verb{url} + \verb https://doi.ieeecomputersociety.org/10.1109/IJCNN.2000.857827 + \endverb + \endentry + \entry{Numerical-Optimization-2006}{book}{}{} + \name{author}{2}{}{% + {{hash=88a342d927bf795b0d92af8a5613da31}{% + family={Nocedal}, + familyi={N\bibinitperiod}, + given={Jorge}, + giveni={J\bibinitperiod}}}% + {{hash=c49efb16d3fa7eef002cc3620d42ab8a}{% + family={Wright}, + familyi={W\bibinitperiod}, + given={Stephen\bibnamedelima J.}, + giveni={S\bibinitperiod\bibinitdelim J\bibinitperiod}}}% + } + \list{location}{1}{% + {New York, NY, USA}% + } + \list{publisher}{1}{% + {Springer}% + } + \strng{namehash}{5b8fd28f6245ae8b0af119560622fa4f} + \strng{fullhash}{5b8fd28f6245ae8b0af119560622fa4f} + \strng{fullhashraw}{5b8fd28f6245ae8b0af119560622fa4f} + \strng{bibnamehash}{5b8fd28f6245ae8b0af119560622fa4f} + \strng{authorbibnamehash}{5b8fd28f6245ae8b0af119560622fa4f} + \strng{authornamehash}{5b8fd28f6245ae8b0af119560622fa4f} + \strng{authorfullhash}{5b8fd28f6245ae8b0af119560622fa4f} + \strng{authorfullhashraw}{5b8fd28f6245ae8b0af119560622fa4f} + \field{sortinit}{2} + \field{sortinithash}{8b555b3791beccb63322c22f3320aa9a} + \field{labelnamesource}{author} + \field{labeltitlesource}{title} + \field{edition}{2e} + \field{title}{Numerical Optimization} + \field{year}{2006} + \endentry + \entry{BenchmarkTools}{article}{}{} + \name{author}{2}{}{% + {{hash=69bd6c70a08a67d2841abb1b24a5ccfa}{% + family={{Chen}}, + familyi={C\bibinitperiod}, + given={Jiahao}, + giveni={J\bibinitperiod}}}% + {{hash=d8a5df654714218c1b828ffab66545f1}{% + family={{Revels}}, + familyi={R\bibinitperiod}, + given={Jarrett}, + giveni={J\bibinitperiod}}}% + } + \strng{namehash}{9162f41ecdba576d7faa1a6d6e27bc25} + \strng{fullhash}{9162f41ecdba576d7faa1a6d6e27bc25} + \strng{fullhashraw}{9162f41ecdba576d7faa1a6d6e27bc25} + \strng{bibnamehash}{9162f41ecdba576d7faa1a6d6e27bc25} + \strng{authorbibnamehash}{9162f41ecdba576d7faa1a6d6e27bc25} + \strng{authornamehash}{9162f41ecdba576d7faa1a6d6e27bc25} + \strng{authorfullhash}{9162f41ecdba576d7faa1a6d6e27bc25} + \strng{authorfullhashraw}{9162f41ecdba576d7faa1a6d6e27bc25} + \field{sortinit}{2} + \field{sortinithash}{8b555b3791beccb63322c22f3320aa9a} + \field{labelnamesource}{author} + \field{labeltitlesource}{title} + \field{eid}{arXiv:1608.04295} + \field{eprintclass}{cs.PF} + \field{eprinttype}{arXiv} + \field{journaltitle}{arXiv e-prints} + \field{month}{8} + \field{title}{{Robust benchmarking in noisy environments}} + \field{year}{2016} + \verb{eprint} + \verb 1608.04295 + \endverb + \keyw{Computer Science - Performance,68N30,B.8.1,D.2.5} + \endentry + \enddatalist +\endrefsection +\endinput + diff --git a/Report/main.bcf b/Report/main.bcf new file mode 100644 index 0000000..eff7efe --- /dev/null +++ b/Report/main.bcf @@ -0,0 +1,2422 @@ + + + + + + output_encoding + utf8 + + + input_encoding + utf8 + + + debug + 0 + + + mincrossrefs + 2 + + + minxrefs + 2 + + + sortcase + 1 + + + sortupper + 1 + + + + + + + alphaothers + + + + + extradatecontext + labelname + labeltitle + + + labelalpha + 0 + + + labelnamespec + shortauthor + author + shorteditor + editor + translator + + + labeltitle + 0 + + + labeltitlespec + shorttitle + title + maintitle + + + labeltitleyear + 0 + + + labeldateparts + 0 + + + labeldatespec + date + year + eventdate + origdate + urldate + nodate + + + julian + 0 + + + gregorianstart + 1582-10-15 + + + maxalphanames + 3 + + + maxbibnames + 6 + + + maxcitenames + 3 + + + maxsortnames + 6 + + + maxitems + 3 + + + minalphanames + 1 + + + minbibnames + 3 + + + mincitenames + 3 + + + minsortnames + 3 + + + minitems + 1 + + + nohashothers + 0 + + + noroman + 0 + + + nosortothers + 0 + + + pluralothers + 0 + + + singletitle + 0 + + + skipbib + 0 + + + skipbiblist + 0 + + + skiplab + 0 + + + sortalphaothers + + + + + sortlocale + english + + + sortingtemplatename + ynt + + + sortsets + 0 + + + uniquelist + false + + + uniquename + false + + + uniqueprimaryauthor + 0 + + + uniquetitle + 0 + + + uniquebaretitle + 0 + + + uniquework + 0 + + + useprefix + 0 + + + useafterword + 1 + + + useannotator + 1 + + + useauthor + 1 + + + usebookauthor + 1 + + + usecommentator + 1 + + + useeditor + 1 + + + useeditora + 1 + + + useeditorb + 1 + + + useeditorc + 1 + + + useforeword + 1 + + + useholder + 1 + + + useintroduction + 1 + + + usenamea + 1 + + + usenameb + 1 + + + usenamec + 1 + + + usetranslator + 0 + + + useshortauthor + 1 + + + useshorteditor + 1 + + + + + + extradatecontext + labelname + labeltitle + + + labelalpha + 0 + + + labelnamespec + shortauthor + author + shorteditor + editor + translator + + + labeltitle + 0 + + + labeltitlespec + shorttitle + title + maintitle + + + labeltitleyear + 0 + + + labeldateparts + 0 + + + labeldatespec + date + year + eventdate + origdate + urldate + nodate + + + maxalphanames + 3 + + + maxbibnames + 6 + + + maxcitenames + 3 + + + maxsortnames + 6 + + + maxitems + 3 + + + minalphanames + 1 + + + minbibnames + 3 + + + mincitenames + 3 + + + minsortnames + 3 + + + minitems + 1 + + + nohashothers + 0 + + + noroman + 0 + + + nosortothers + 0 + + + singletitle + 0 + + + skipbib + 0 + + + skipbiblist + 0 + + + skiplab + 0 + + + uniquelist + false + + + uniquename + false + + + uniqueprimaryauthor + 0 + + + uniquetitle + 0 + + + uniquebaretitle + 0 + + + uniquework + 0 + + + useprefix + 0 + + + useafterword + 1 + + + useannotator + 1 + + + useauthor + 1 + + + usebookauthor + 1 + + + usecommentator + 1 + + + useeditor + 1 + + + useeditora + 1 + + + useeditorb + 1 + + + useeditorc + 1 + + + useforeword + 1 + + + useholder + 1 + + + useintroduction + 1 + + + usenamea + 1 + + + usenameb + 1 + + + usenamec + 1 + + + usetranslator + 0 + + + useshortauthor + 1 + + + useshorteditor + 1 + + + + + datamodel + labelalphanametemplate + labelalphatemplate + inheritance + translit + uniquenametemplate + namehashtemplate + sortingnamekeytemplate + sortingtemplate + extradatespec + extradatecontext + labelnamespec + labeltitlespec + labeldatespec + controlversion + alphaothers + sortalphaothers + presort + texencoding + bibencoding + sortingtemplatename + sortlocale + language + autolang + langhook + indexing + hyperref + backrefsetstyle + block + pagetracker + citecounter + citetracker + ibidtracker + idemtracker + opcittracker + loccittracker + labeldate + labeltime + dateera + date + time + eventdate + eventtime + origdate + origtime + urldate + urltime + alldatesusetime + alldates + alltimes + gregorianstart + autocite + notetype + uniquelist + uniquename + refsection + refsegment + citereset + sortlos + babel + datelabel + backrefstyle + arxiv + familyinits + giveninits + prefixinits + suffixinits + useafterword + useannotator + useauthor + usebookauthor + usecommentator + useeditor + useeditora + useeditorb + useeditorc + useforeword + useholder + useintroduction + usenamea + usenameb + usenamec + usetranslator + useshortauthor + useshorteditor + debug + loadfiles + safeinputenc + sortcase + sortupper + terseinits + abbreviate + dateabbrev + clearlang + sortcites + sortsets + backref + backreffloats + trackfloats + parentracker + labeldateusetime + datecirca + dateuncertain + dateusetime + eventdateusetime + origdateusetime + urldateusetime + julian + datezeros + timezeros + timezones + seconds + autopunct + punctfont + labelnumber + labelalpha + labeltitle + labeltitleyear + labeldateparts + pluralothers + nohashothers + nosortothers + noroman + singletitle + uniquetitle + uniquebaretitle + uniquework + uniqueprimaryauthor + defernumbers + locallabelwidth + bibwarn + useprefix + skipbib + skipbiblist + skiplab + dataonly + defernums + firstinits + sortfirstinits + sortgiveninits + labelyear + isbn + url + doi + eprint + related + subentry + dashed + subentrycomp + bibtexcaseprotection + mincrossrefs + minxrefs + maxnames + minnames + maxbibnames + minbibnames + maxcitenames + mincitenames + maxsortnames + minsortnames + maxitems + minitems + maxalphanames + minalphanames + maxparens + dateeraauto + + + alphaothers + sortalphaothers + presort + indexing + citetracker + ibidtracker + idemtracker + opcittracker + loccittracker + uniquelist + uniquename + familyinits + giveninits + prefixinits + suffixinits + useafterword + useannotator + useauthor + usebookauthor + usecommentator + useeditor + useeditora + useeditorb + useeditorc + useforeword + useholder + useintroduction + usenamea + usenameb + usenamec + usetranslator + useshortauthor + useshorteditor + terseinits + abbreviate + dateabbrev + clearlang + labelnumber + labelalpha + labeltitle + labeltitleyear + labeldateparts + nohashothers + nosortothers + noroman + singletitle + uniquetitle + uniquebaretitle + uniquework + uniqueprimaryauthor + useprefix + skipbib + skipbiblist + skiplab + dataonly + skiplos + labelyear + isbn + url + doi + eprint + related + subentry + subentrycomp + bibtexcaseprotection + labelalphatemplate + translit + sortexclusion + sortinclusion + extradatecontext + labelnamespec + labeltitlespec + labeldatespec + maxnames + minnames + maxbibnames + minbibnames + maxcitenames + mincitenames + maxsortnames + minsortnames + maxitems + minitems + maxalphanames + minalphanames + + + noinherit + nametemplates + labelalphanametemplatename + uniquenametemplatename + namehashtemplatename + sortingnamekeytemplatename + presort + indexing + citetracker + ibidtracker + idemtracker + opcittracker + loccittracker + uniquelist + uniquename + familyinits + giveninits + prefixinits + suffixinits + useafterword + useannotator + useauthor + usebookauthor + usecommentator + useeditor + useeditora + useeditorb + useeditorc + useforeword + useholder + useintroduction + usenamea + usenameb + usenamec + usetranslator + useshortauthor + useshorteditor + terseinits + abbreviate + dateabbrev + clearlang + labelnumber + labelalpha + labeltitle + labeltitleyear + labeldateparts + nohashothers + nosortothers + noroman + singletitle + uniquetitle + uniquebaretitle + uniquework + uniqueprimaryauthor + useprefix + skipbib + skipbiblist + skiplab + dataonly + skiplos + isbn + url + doi + eprint + related + subentry + subentrycomp + bibtexcaseprotection + maxnames + minnames + maxbibnames + minbibnames + maxcitenames + mincitenames + maxsortnames + minsortnames + maxitems + minitems + maxalphanames + minalphanames + + + nametemplates + labelalphanametemplatename + uniquenametemplatename + namehashtemplatename + sortingnamekeytemplatename + uniquelist + uniquename + familyinits + giveninits + prefixinits + suffixinits + terseinits + nohashothers + nosortothers + useprefix + + + nametemplates + labelalphanametemplatename + uniquenametemplatename + namehashtemplatename + sortingnamekeytemplatename + uniquename + familyinits + giveninits + prefixinits + suffixinits + terseinits + useprefix + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + prefix + family + + + + + shorthand + label + labelname + labelname + + + year + + + + + + labelyear + year + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + prefix + family + given + + + + family + given + prefix + suffix + + + + + prefix + family + + + given + + + suffix + + + prefix + + + mm + + + + sf,sm,sn,pf,pm,pn,pp + family,given,prefix,suffix + boolean,integer,string,xml + default,transliteration,transcription,translation + + + article + artwork + audio + bibnote + book + bookinbook + booklet + collection + commentary + customa + customb + customc + customd + custome + customf + dataset + inbook + incollection + inproceedings + inreference + image + jurisdiction + legal + legislation + letter + manual + misc + movie + music + mvcollection + mvreference + mvproceedings + mvbook + online + patent + performance + periodical + proceedings + reference + report + review + set + software + standard + suppbook + suppcollection + suppperiodical + thesis + unpublished + video + xdata + + + sortyear + volume + volumes + abstract + addendum + annotation + booksubtitle + booktitle + booktitleaddon + chapter + edition + eid + entrysubtype + eprintclass + eprinttype + eventtitle + eventtitleaddon + gender + howpublished + indexsorttitle + indextitle + isan + isbn + ismn + isrn + issn + issue + issuesubtitle + issuetitle + issuetitleaddon + iswc + journalsubtitle + journaltitle + journaltitleaddon + label + langid + langidopts + library + mainsubtitle + maintitle + maintitleaddon + nameaddon + note + number + origtitle + pagetotal + part + relatedstring + relatedtype + reprinttitle + series + shorthandintro + subtitle + title + titleaddon + usera + userb + userc + userd + usere + userf + venue + version + shorthand + shortjournal + shortseries + shorttitle + sorttitle + sortshorthand + sortkey + presort + institution + lista + listb + listc + listd + liste + listf + location + organization + origlocation + origpublisher + publisher + afterword + annotator + author + bookauthor + commentator + editor + editora + editorb + editorc + foreword + holder + introduction + namea + nameb + namec + translator + shortauthor + shorteditor + sortname + authortype + editoratype + editorbtype + editorctype + editortype + bookpagination + nameatype + namebtype + namectype + pagination + pubstate + type + language + origlanguage + crossref + xref + date + endyear + year + month + day + hour + minute + second + timezone + yeardivision + endmonth + endday + endhour + endminute + endsecond + endtimezone + endyeardivision + eventdate + eventendyear + eventyear + eventmonth + eventday + eventhour + eventminute + eventsecond + eventtimezone + eventyeardivision + eventendmonth + eventendday + eventendhour + eventendminute + eventendsecond + eventendtimezone + eventendyeardivision + origdate + origendyear + origyear + origmonth + origday + orighour + origminute + origsecond + origtimezone + origyeardivision + origendmonth + origendday + origendhour + origendminute + origendsecond + origendtimezone + origendyeardivision + urldate + urlendyear + urlyear + urlmonth + urlday + urlhour + urlminute + urlsecond + urltimezone + urlyeardivision + urlendmonth + urlendday + urlendhour + urlendminute + urlendsecond + urlendtimezone + urlendyeardivision + doi + eprint + file + verba + verbb + verbc + url + xdata + ids + entryset + related + keywords + options + relatedoptions + pages + execute + + + abstract + annotation + authortype + bookpagination + crossref + day + doi + eprint + eprintclass + eprinttype + endday + endhour + endminute + endmonth + endsecond + endtimezone + endyear + endyeardivision + entryset + entrysubtype + execute + file + gender + hour + ids + indextitle + indexsorttitle + isan + ismn + iswc + keywords + label + langid + langidopts + library + lista + listb + listc + listd + liste + listf + minute + month + namea + nameb + namec + nameatype + namebtype + namectype + nameaddon + options + origday + origendday + origendhour + origendminute + origendmonth + origendsecond + origendtimezone + origendyear + origendyeardivision + orighour + origminute + origmonth + origsecond + origtimezone + origyear + origyeardivision + origlocation + origpublisher + origtitle + pagination + presort + related + relatedoptions + relatedstring + relatedtype + second + shortauthor + shorteditor + shorthand + shorthandintro + shortjournal + shortseries + shorttitle + sortkey + sortname + sortshorthand + sorttitle + sortyear + timezone + url + urlday + urlendday + urlendhour + urlendminute + urlendmonth + urlendsecond + urlendtimezone + urlendyear + urlhour + urlminute + urlmonth + urlsecond + urltimezone + urlyear + usera + userb + userc + userd + usere + userf + verba + verbb + verbc + xdata + xref + year + yeardivision + + + set + entryset + + + article + addendum + annotator + author + commentator + editor + editora + editorb + editorc + editortype + editoratype + editorbtype + editorctype + eid + issn + issue + issuetitle + issuesubtitle + issuetitleaddon + journalsubtitle + journaltitle + journaltitleaddon + language + note + number + origlanguage + pages + pubstate + series + subtitle + title + titleaddon + translator + version + volume + + + bibnote + note + + + book + author + addendum + afterword + annotator + chapter + commentator + edition + editor + editora + editorb + editorc + editortype + editoratype + editorbtype + editorctype + eid + foreword + introduction + isbn + language + location + maintitle + maintitleaddon + mainsubtitle + note + number + origlanguage + pages + pagetotal + part + publisher + pubstate + series + subtitle + title + titleaddon + translator + volume + volumes + + + mvbook + addendum + afterword + annotator + author + commentator + edition + editor + editora + editorb + editorc + editortype + editoratype + editorbtype + editorctype + foreword + introduction + isbn + language + location + note + number + origlanguage + pagetotal + publisher + pubstate + series + subtitle + title + titleaddon + translator + volume + volumes + + + inbook + bookinbook + suppbook + addendum + afterword + annotator + author + booktitle + bookauthor + booksubtitle + booktitleaddon + chapter + commentator + edition + editor + editora + editorb + editorc + editortype + editoratype + editorbtype + editorctype + eid + foreword + introduction + isbn + language + location + mainsubtitle + maintitle + maintitleaddon + note + number + origlanguage + part + publisher + pages + pubstate + series + subtitle + title + titleaddon + translator + volume + volumes + + + booklet + addendum + author + chapter + editor + editortype + eid + howpublished + language + location + note + pages + pagetotal + pubstate + subtitle + title + titleaddon + type + + + collection + reference + addendum + afterword + annotator + chapter + commentator + edition + editor + editora + editorb + editorc + editortype + editoratype + editorbtype + editorctype + eid + foreword + introduction + isbn + language + location + mainsubtitle + maintitle + maintitleaddon + note + number + origlanguage + pages + pagetotal + part + publisher + pubstate + series + subtitle + title + titleaddon + translator + volume + volumes + + + mvcollection + mvreference + addendum + afterword + annotator + author + commentator + edition + editor + editora + editorb + editorc + editortype + editoratype + editorbtype + editorctype + foreword + introduction + isbn + language + location + note + number + origlanguage + publisher + pubstate + subtitle + title + titleaddon + translator + volume + volumes + + + incollection + suppcollection + inreference + addendum + afterword + annotator + author + booksubtitle + booktitle + booktitleaddon + chapter + commentator + edition + editor + editora + editorb + editorc + editortype + editoratype + editorbtype + editorctype + eid + foreword + introduction + isbn + language + location + mainsubtitle + maintitle + maintitleaddon + note + number + origlanguage + pages + part + publisher + pubstate + series + subtitle + title + titleaddon + translator + volume + volumes + + + dataset + addendum + author + edition + editor + editortype + language + location + note + number + organization + publisher + pubstate + series + subtitle + title + titleaddon + type + version + + + manual + addendum + author + chapter + edition + editor + editortype + eid + isbn + language + location + note + number + organization + pages + pagetotal + publisher + pubstate + series + subtitle + title + titleaddon + type + version + + + misc + software + addendum + author + editor + editortype + howpublished + language + location + note + organization + pubstate + subtitle + title + titleaddon + type + version + + + online + addendum + author + editor + editortype + language + note + organization + pubstate + subtitle + title + titleaddon + version + + + patent + addendum + author + holder + location + note + number + pubstate + subtitle + title + titleaddon + type + version + + + periodical + addendum + editor + editora + editorb + editorc + editortype + editoratype + editorbtype + editorctype + issn + issue + issuesubtitle + issuetitle + issuetitleaddon + language + note + number + pubstate + series + subtitle + title + titleaddon + volume + yeardivision + + + mvproceedings + addendum + editor + editortype + eventday + eventendday + eventendhour + eventendminute + eventendmonth + eventendsecond + eventendtimezone + eventendyear + eventendyeardivision + eventhour + eventminute + eventmonth + eventsecond + eventtimezone + eventyear + eventyeardivision + eventtitle + eventtitleaddon + isbn + language + location + note + number + organization + pagetotal + publisher + pubstate + series + subtitle + title + titleaddon + venue + volumes + + + proceedings + addendum + chapter + editor + editortype + eid + eventday + eventendday + eventendhour + eventendminute + eventendmonth + eventendsecond + eventendtimezone + eventendyear + eventendyeardivision + eventhour + eventminute + eventmonth + eventsecond + eventtimezone + eventyear + eventyeardivision + eventtitle + eventtitleaddon + isbn + language + location + mainsubtitle + maintitle + maintitleaddon + note + number + organization + pages + pagetotal + part + publisher + pubstate + series + subtitle + title + titleaddon + venue + volume + volumes + + + inproceedings + addendum + author + booksubtitle + booktitle + booktitleaddon + chapter + editor + editortype + eid + eventday + eventendday + eventendhour + eventendminute + eventendmonth + eventendsecond + eventendtimezone + eventendyear + eventendyeardivision + eventhour + eventminute + eventmonth + eventsecond + eventtimezone + eventyear + eventyeardivision + eventtitle + eventtitleaddon + isbn + language + location + mainsubtitle + maintitle + maintitleaddon + note + number + organization + pages + part + publisher + pubstate + series + subtitle + title + titleaddon + venue + volume + volumes + + + report + addendum + author + chapter + eid + institution + isrn + language + location + note + number + pages + pagetotal + pubstate + subtitle + title + titleaddon + type + version + + + thesis + addendum + author + chapter + eid + institution + language + location + note + pages + pagetotal + pubstate + subtitle + title + titleaddon + type + + + unpublished + addendum + author + eventday + eventendday + eventendhour + eventendminute + eventendmonth + eventendsecond + eventendtimezone + eventendyear + eventendyeardivision + eventhour + eventminute + eventmonth + eventsecond + eventtimezone + eventyear + eventyeardivision + eventtitle + eventtitleaddon + howpublished + language + location + note + pubstate + subtitle + title + titleaddon + type + venue + + + abstract + addendum + afterword + annotator + author + bookauthor + booksubtitle + booktitle + booktitleaddon + chapter + commentator + editor + editora + editorb + editorc + foreword + holder + institution + introduction + issuesubtitle + issuetitle + issuetitleaddon + journalsubtitle + journaltitle + journaltitleaddon + location + mainsubtitle + maintitle + maintitleaddon + nameaddon + note + organization + origlanguage + origlocation + origpublisher + origtitle + part + publisher + relatedstring + series + shortauthor + shorteditor + shorthand + shortjournal + shortseries + shorttitle + sortname + sortshorthand + sorttitle + subtitle + title + titleaddon + translator + venue + + + article + book + inbook + bookinbook + suppbook + booklet + collection + incollection + suppcollection + manual + misc + mvbook + mvcollection + online + patent + periodical + suppperiodical + proceedings + inproceedings + reference + inreference + report + set + thesis + unpublished + + + date + year + + + + + set + + entryset + + + + article + + author + journaltitle + title + + + + book + mvbook + + author + title + + + + inbook + bookinbook + suppbook + + author + title + booktitle + + + + booklet + + + author + editor + + title + + + + collection + reference + mvcollection + mvreference + + editor + title + + + + incollection + suppcollection + inreference + + author + editor + title + booktitle + + + + dataset + + title + + + + manual + + title + + + + misc + software + + title + + + + online + + title + + url + doi + eprint + + + + + patent + + author + title + number + + + + periodical + + editor + title + + + + proceedings + mvproceedings + + title + + + + inproceedings + + author + title + booktitle + + + + report + + author + title + type + institution + + + + thesis + + author + title + type + institution + + + + unpublished + + author + title + + + + + isbn + + + issn + + + ismn + + + gender + + + + + + + document.bib + + + Numerical-Optimization-2006 + convergence_lbfgs + BenchmarkTools + Numerical-Optimization-2006 + Dogleg + + + + + presort + + + sortkey + + + sortyear + year + 9999 + + + sortname + author + editor + translator + sorttitle + title + + + sorttitle + title + + + + + + diff --git a/Report/main.blg b/Report/main.blg new file mode 100644 index 0000000..67bddc8 --- /dev/null +++ b/Report/main.blg @@ -0,0 +1,15 @@ +[0] Config.pm:308> INFO - This is Biber 2.20 +[0] Config.pm:311> INFO - Logfile is 'main.blg' +[30] biber-darwin:340> INFO - === Mar Lug 30, 2024, 14:41:06 +[36] Biber.pm:420> INFO - Reading 'main.bcf' +[68] Biber.pm:994> INFO - Found 4 citekeys in bib section 0 +[75] Biber.pm:4463> INFO - Processing section 0 +[79] Biber.pm:4654> INFO - Looking for bibtex file 'document.bib' for section 0 +[80] bibtex.pm:1713> INFO - LaTeX decoding ... +[83] bibtex.pm:1519> INFO - Found BibTeX data source 'document.bib' +[106] UCollate.pm:68> INFO - Overriding locale 'en-US' defaults 'normalization = NFD' with 'normalization = prenormalized' +[106] UCollate.pm:68> INFO - Overriding locale 'en-US' defaults 'variable = shifted' with 'variable = non-ignorable' +[106] Biber.pm:4283> INFO - Sorting list 'ynt/global//global/global/global' of type 'entry' with template 'ynt' and locale 'en-US' +[106] Biber.pm:4289> INFO - No sort tailoring available for locale 'en-US' +[110] bbl.pm:676> INFO - Writing 'main.bbl' with encoding 'UTF-8' +[111] bbl.pm:779> INFO - Output to main.bbl diff --git a/Report/main.log b/Report/main.log new file mode 100644 index 0000000..0e63ae2 --- /dev/null +++ b/Report/main.log @@ -0,0 +1,2099 @@ +This is pdfTeX, Version 3.141592653-2.6-1.40.26 (TeX Live 2024) (preloaded format=pdflatex 2024.6.16) 30 JUL 2024 14:40 +entering extended mode + \write18 enabled. + file:line:error style messages enabled. + %&-line parsing enabled. +**main.tex +(./main.tex +LaTeX2e <2024-06-01> patch level 1 +L3 programming layer <2024-05-27> +(/usr/local/texlive/2024/texmf-dist/tex/latex/base/report.cls +Document Class: report 2024/02/08 v1.4n Standard LaTeX document class +(/usr/local/texlive/2024/texmf-dist/tex/latex/base/size12.clo +File: size12.clo 2024/02/08 v1.4n Standard LaTeX file (size option) +) +\c@part=\count194 +\c@chapter=\count195 +\c@section=\count196 +\c@subsection=\count197 +\c@subsubsection=\count198 +\c@paragraph=\count199 +\c@subparagraph=\count266 +\c@figure=\count267 +\c@table=\count268 +\abovecaptionskip=\skip49 +\belowcaptionskip=\skip50 +\bibindent=\dimen141 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/geometry/geometry.sty +Package: geometry 2020/01/02 v5.9 Page Geometry + +(/usr/local/texlive/2024/texmf-dist/tex/latex/graphics/keyval.sty +Package: keyval 2022/05/29 v1.15 key=value parser (DPC) +\KV@toks@=\toks17 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/iftex/ifvtex.sty +Package: ifvtex 2019/10/25 v1.7 ifvtex legacy package. Use iftex instead. + +(/usr/local/texlive/2024/texmf-dist/tex/generic/iftex/iftex.sty +Package: iftex 2022/02/03 v1.0f TeX engine tests +)) +\Gm@cnth=\count269 +\Gm@cntv=\count270 +\c@Gm@tempcnt=\count271 +\Gm@bindingoffset=\dimen142 +\Gm@wd@mp=\dimen143 +\Gm@odd@mp=\dimen144 +\Gm@even@mp=\dimen145 +\Gm@layoutwidth=\dimen146 +\Gm@layoutheight=\dimen147 +\Gm@layouthoffset=\dimen148 +\Gm@layoutvoffset=\dimen149 +\Gm@dimlist=\toks18 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/base/inputenc.sty +Package: inputenc 2024/02/08 v1.3d Input encoding file +\inpenc@prehook=\toks19 +\inpenc@posthook=\toks20 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/babel/babel.sty +Package: babel 2024/06/02 v24.6 The Babel package +\babel@savecnt=\count272 +\U@D=\dimen150 +\l@unhyphenated=\language89 + +(/usr/local/texlive/2024/texmf-dist/tex/generic/babel/txtbabel.def) +\bbl@readstream=\read2 +\bbl@dirlevel=\count273 + +(/usr/local/texlive/2024/texmf-dist/tex/generic/babel-english/english.ldf +Language: english 2017/06/06 v3.3r English support from the babel system +Package babel Info: Hyphen rules for 'canadian' set to \l@english +(babel) (\language0). Reported on input line 102. +Package babel Info: Hyphen rules for 'australian' set to \l@ukenglish +(babel) (\language23). Reported on input line 105. +Package babel Info: Hyphen rules for 'newzealand' set to \l@ukenglish +(babel) (\language23). Reported on input line 108. +)) +(/usr/local/texlive/2024/texmf-dist/tex/generic/babel/locale/en/babel-english.t +ex +Package babel Info: Importing font and identification data for english +(babel) from babel-en.ini. Reported on input line 11. +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/koma-script/scrlayer-scrpage.sty +Package: scrlayer-scrpage 2023/07/07 v3.41 KOMA-Script package (end user interf +ace for scrlayer) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/koma-script/scrlayer.sty +Package: scrlayer 2023/07/07 v3.41 KOMA-Script package (defining layers and pag +e styles) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/koma-script/scrkbase.sty +Package: scrkbase 2023/07/07 v3.41 KOMA-Script package (KOMA-Script-dependent b +asics and keyval usage) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/koma-script/scrbase.sty +Package: scrbase 2023/07/07 v3.41 KOMA-Script package (KOMA-Script-independent +basics and keyval usage) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/koma-script/scrlfile.sty +Package: scrlfile 2023/07/07 v3.41 KOMA-Script package (file load hooks) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/koma-script/scrlfile-hook.sty +Package: scrlfile-hook 2023/07/07 v3.41 KOMA-Script package (using LaTeX hooks) + + +(/usr/local/texlive/2024/texmf-dist/tex/latex/koma-script/scrlogo.sty +Package: scrlogo 2023/07/07 v3.41 KOMA-Script package (logo) +))) +Applying: [2021/05/01] Usage of raw or classic option list on input line 252. +Already applied: [0000/00/00] Usage of raw or classic option list on input line + 368. +)) +\footheight=\skip51 +Package scrlayer Info: patching LaTeX kernel macro \pagestyle on input line 218 +3. +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex/biblatex.sty +Package: biblatex 2024/03/21 v3.20 programmable bibliographies (PK/MW) + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty +Package: pdftexcmds 2020-06-27 v0.33 Utility functions of pdfTeX for LuaTeX (HO +) + +(/usr/local/texlive/2024/texmf-dist/tex/generic/infwarerr/infwarerr.sty +Package: infwarerr 2019/12/03 v1.5 Providing info/warning/error messages (HO) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty +Package: ltxcmds 2023-12-04 v1.26 LaTeX kernel commands for general use (HO) +) +Package pdftexcmds Info: \pdf@primitive is available. +Package pdftexcmds Info: \pdf@ifprimitive is available. +Package pdftexcmds Info: \pdfdraftmode found. +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/etoolbox/etoolbox.sty +Package: etoolbox 2020/10/05 v2.5k e-TeX tools for LaTeX (JAW) +\etb@tempcnta=\count274 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/kvoptions/kvoptions.sty +Package: kvoptions 2022-06-15 v3.15 Key value format for package options (HO) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty +Package: kvsetkeys 2022-10-05 v1.19 Key value parser (HO) +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/logreq/logreq.sty +Package: logreq 2010/08/04 v1.0 xml request logger +\lrq@indent=\count275 + +(/usr/local/texlive/2024/texmf-dist/tex/latex/logreq/logreq.def +File: logreq.def 2010/08/04 v1.0 logreq spec v1.0 +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/base/ifthen.sty +Package: ifthen 2024/03/16 v1.1e Standard LaTeX ifthen package (DPC) +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/url/url.sty +\Urlmuskip=\muskip17 +Package: url 2013/09/16 ver 3.4 Verb mode for urls, etc. +) +\c@tabx@nest=\count276 +\c@listtotal=\count277 +\c@listcount=\count278 +\c@liststart=\count279 +\c@liststop=\count280 +\c@citecount=\count281 +\c@citetotal=\count282 +\c@multicitecount=\count283 +\c@multicitetotal=\count284 +\c@instcount=\count285 +\c@maxnames=\count286 +\c@minnames=\count287 +\c@maxitems=\count288 +\c@minitems=\count289 +\c@citecounter=\count290 +\c@maxcitecounter=\count291 +\c@savedcitecounter=\count292 +\c@uniquelist=\count293 +\c@uniquename=\count294 +\c@refsection=\count295 +\c@refsegment=\count296 +\c@maxextratitle=\count297 +\c@maxextratitleyear=\count298 +\c@maxextraname=\count299 +\c@maxextradate=\count300 +\c@maxextraalpha=\count301 +\c@abbrvpenalty=\count302 +\c@highnamepenalty=\count303 +\c@lownamepenalty=\count304 +\c@maxparens=\count305 +\c@parenlevel=\count306 +\blx@tempcnta=\count307 +\blx@tempcntb=\count308 +\blx@tempcntc=\count309 +\c@blx@maxsection=\count310 +\blx@maxsegment@0=\count311 +\blx@notetype=\count312 +\blx@parenlevel@text=\count313 +\blx@parenlevel@foot=\count314 +\blx@sectionciteorder@0=\count315 +\blx@sectionciteorderinternal@0=\count316 +\blx@entrysetcounter=\count317 +\blx@biblioinstance=\count318 +\labelnumberwidth=\skip52 +\labelalphawidth=\skip53 +\biblabelsep=\skip54 +\bibitemsep=\skip55 +\bibnamesep=\skip56 +\bibinitsep=\skip57 +\bibparsep=\skip58 +\bibhang=\skip59 +\blx@bcfin=\read3 +\blx@bcfout=\write3 +\blx@langwohyphens=\language90 +\c@mincomprange=\count319 +\c@maxcomprange=\count320 +\c@mincompwidth=\count321 +Package biblatex Info: Trying to load biblatex default data model... +Package biblatex Info: ... file 'blx-dm.def' found. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex/blx-dm.def +File: blx-dm.def 2024/03/21 v3.20 biblatex datamodel (PK/MW) +) +Package biblatex Info: Trying to load biblatex style data model... +Package biblatex Info: ... file 'ieee.dbx' not found. +Package biblatex Info: Trying to load biblatex custom data model... +Package biblatex Info: ... file 'biblatex-dm.cfg' not found. +\c@afterword=\count322 +\c@savedafterword=\count323 +\c@annotator=\count324 +\c@savedannotator=\count325 +\c@author=\count326 +\c@savedauthor=\count327 +\c@bookauthor=\count328 +\c@savedbookauthor=\count329 +\c@commentator=\count330 +\c@savedcommentator=\count331 +\c@editor=\count332 +\c@savededitor=\count333 +\c@editora=\count334 +\c@savededitora=\count335 +\c@editorb=\count336 +\c@savededitorb=\count337 +\c@editorc=\count338 +\c@savededitorc=\count339 +\c@foreword=\count340 +\c@savedforeword=\count341 +\c@holder=\count342 +\c@savedholder=\count343 +\c@introduction=\count344 +\c@savedintroduction=\count345 +\c@namea=\count346 +\c@savednamea=\count347 +\c@nameb=\count348 +\c@savednameb=\count349 +\c@namec=\count350 +\c@savednamec=\count351 +\c@translator=\count352 +\c@savedtranslator=\count353 +\c@shortauthor=\count354 +\c@savedshortauthor=\count355 +\c@shorteditor=\count356 +\c@savedshorteditor=\count357 +\c@labelname=\count358 +\c@savedlabelname=\count359 +\c@institution=\count360 +\c@savedinstitution=\count361 +\c@lista=\count362 +\c@savedlista=\count363 +\c@listb=\count364 +\c@savedlistb=\count365 +\c@listc=\count366 +\c@savedlistc=\count367 +\c@listd=\count368 +\c@savedlistd=\count369 +\c@liste=\count370 +\c@savedliste=\count371 +\c@listf=\count372 +\c@savedlistf=\count373 +\c@location=\count374 +\c@savedlocation=\count375 +\c@organization=\count376 +\c@savedorganization=\count377 +\c@origlocation=\count378 +\c@savedoriglocation=\count379 +\c@origpublisher=\count380 +\c@savedorigpublisher=\count381 +\c@publisher=\count382 +\c@savedpublisher=\count383 +\c@language=\count384 +\c@savedlanguage=\count385 +\c@origlanguage=\count386 +\c@savedoriglanguage=\count387 +\c@pageref=\count388 +\c@savedpageref=\count389 +\shorthandwidth=\skip60 +\shortjournalwidth=\skip61 +\shortserieswidth=\skip62 +\shorttitlewidth=\skip63 +\shortauthorwidth=\skip64 +\shorteditorwidth=\skip65 +\locallabelnumberwidth=\skip66 +\locallabelalphawidth=\skip67 +\localshorthandwidth=\skip68 +\localshortjournalwidth=\skip69 +\localshortserieswidth=\skip70 +\localshorttitlewidth=\skip71 +\localshortauthorwidth=\skip72 +\localshorteditorwidth=\skip73 +Package biblatex Info: Trying to load compatibility code... +Package biblatex Info: ... file 'blx-compat.def' found. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex/blx-compat.def +File: blx-compat.def 2024/03/21 v3.20 biblatex compatibility (PK/MW) +) +Package biblatex Info: Trying to load generic definitions... +Package biblatex Info: ... file 'biblatex.def' found. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex/biblatex.def +File: biblatex.def 2024/03/21 v3.20 biblatex compatibility (PK/MW) +\c@textcitecount=\count390 +\c@textcitetotal=\count391 +\c@textcitemaxnames=\count392 +\c@biburlbigbreakpenalty=\count393 +\c@biburlbreakpenalty=\count394 +\c@biburlnumpenalty=\count395 +\c@biburlucpenalty=\count396 +\c@biburllcpenalty=\count397 +\biburlbigskip=\muskip18 +\biburlnumskip=\muskip19 +\biburlucskip=\muskip20 +\biburllcskip=\muskip21 +\c@smartand=\count398 +) +Package biblatex Info: Trying to load bibliography style 'ieee'... +Package biblatex Info: ... file 'ieee.bbx' found. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex-ieee/ieee.bbx +File: ieee.bbx 2021/12/06 v1.3f biblatex bibliography style +Package biblatex Info: Trying to load bibliography style 'numeric-comp'... +Package biblatex Info: ... file 'numeric-comp.bbx' found. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex/bbx/numeric-comp.bbx +File: numeric-comp.bbx 2024/03/21 v3.20 biblatex bibliography style (PK/MW) +Package biblatex Info: Trying to load bibliography style 'numeric'... +Package biblatex Info: ... file 'numeric.bbx' found. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex/bbx/numeric.bbx +File: numeric.bbx 2024/03/21 v3.20 biblatex bibliography style (PK/MW) +Package biblatex Info: Trying to load bibliography style 'standard'... +Package biblatex Info: ... file 'standard.bbx' found. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex/bbx/standard.bbx +File: standard.bbx 2024/03/21 v3.20 biblatex bibliography style (PK/MW) +\c@bbx:relatedcount=\count399 +\c@bbx:relatedtotal=\count400 +)))) +Package biblatex Info: Trying to load citation style 'ieee'... +Package biblatex Info: ... file 'ieee.cbx' found. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex-ieee/ieee.cbx +File: ieee.cbx 2021/12/06 v1.3f biblatex citation style +Package biblatex Info: Trying to load citation style 'numeric-comp'... +Package biblatex Info: ... file 'numeric-comp.cbx' found. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex/cbx/numeric-comp.cbx +File: numeric-comp.cbx 2024/03/21 v3.20 biblatex citation style (PK/MW) +\c@cbx@tempcnta=\count401 +\c@cbx@tempcntb=\count402 +\c@cbx@tempcntc=\count403 +\c@cbx@tempcntd=\count404 +Package biblatex Info: Redefining '\cite'. +Package biblatex Info: Redefining '\parencite'. +Package biblatex Info: Redefining '\footcite'. +Package biblatex Info: Redefining '\footcitetext'. +Package biblatex Info: Redefining '\smartcite'. +Package biblatex Info: Redefining '\supercite'. +Package biblatex Info: Redefining '\textcite'. +Package biblatex Info: Redefining '\textcites'. +Package biblatex Info: Redefining '\cites'. +Package biblatex Info: Redefining '\parencites'. +Package biblatex Info: Redefining '\smartcites'. +) +Package biblatex Info: Redefining '\cite'. +Package biblatex Info: Redefining '\supercite'. +Package biblatex Info: Redefining '\cites'. +) +Package biblatex Info: Trying to load configuration file... +Package biblatex Info: ... file 'biblatex.cfg' found. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex/biblatex.cfg +File: biblatex.cfg +) +Package biblatex Info: Input encoding 'utf8' detected. +Package biblatex Info: Document encoding is UTF8 .... + +(/usr/local/texlive/2024/texmf-dist/tex/latex/l3kernel/expl3.sty +Package: expl3 2024-05-27 L3 programming layer (loader) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def +File: l3backend-pdftex.def 2024-05-08 L3 backend support: PDF output (pdfTeX) +\l__color_backend_stack_int=\count405 +\l__pdf_internal_box=\box52 +)) +Package biblatex Info: ... and expl3 +(biblatex) 2024-05-27 L3 programming layer (loader) +(biblatex) is new enough (at least 2020/04/06), +(biblatex) setting 'casechanger=expl3'. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex/blx-case-expl3.sty +(/usr/local/texlive/2024/texmf-dist/tex/latex/l3packages/xparse/xparse.sty +Package: xparse 2024-05-08 L3 Experimental document command parser +) +Package: blx-case-expl3 2024/03/21 v3.20 expl3 case changing code for biblatex +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/import/import.sty +Package: import 2020/04/01 v 6.2 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/graphics/graphicx.sty +Package: graphicx 2021/09/16 v1.2d Enhanced LaTeX Graphics (DPC,SPQR) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/graphics/graphics.sty +Package: graphics 2024/05/23 v1.4g Standard LaTeX Graphics (DPC,SPQR) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/graphics/trig.sty +Package: trig 2023/12/02 v1.11 sin cos tan (DPC) +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/graphics-cfg/graphics.cfg +File: graphics.cfg 2016/06/04 v1.11 sample graphics configuration +) +Package graphics Info: Driver file: pdftex.def on input line 106. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/graphics-def/pdftex.def +File: pdftex.def 2024/04/13 v1.2c Graphics/color driver for pdftex +)) +\Gin@req@height=\dimen151 +\Gin@req@width=\dimen152 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/float/float.sty +Package: float 2001/11/08 v1.3d Float enhancements (AL) +\c@float@type=\count406 +\float@exts=\toks21 +\float@box=\box53 +\@float@everytoks=\toks22 +\@floatcapt=\box54 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/amsfonts/amssymb.sty +Package: amssymb 2013/01/14 v3.01 AMS font symbols + +(/usr/local/texlive/2024/texmf-dist/tex/latex/amsfonts/amsfonts.sty +Package: amsfonts 2013/01/14 v3.01 Basic AMSFonts support +\@emptytoks=\toks23 +\symAMSa=\mathgroup4 +\symAMSb=\mathgroup5 +LaTeX Font Info: Redeclaring math symbol \hbar on input line 98. +LaTeX Font Info: Overwriting math alphabet `\mathfrak' in version `bold' +(Font) U/euf/m/n --> U/euf/b/n on input line 106. +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/amsmath/amsmath.sty +Package: amsmath 2024/05/23 v2.17q AMS math features +\@mathmargin=\skip74 + +For additional information on amsmath, use the `?' option. +(/usr/local/texlive/2024/texmf-dist/tex/latex/amsmath/amstext.sty +Package: amstext 2021/08/26 v2.01 AMS text + +(/usr/local/texlive/2024/texmf-dist/tex/latex/amsmath/amsgen.sty +File: amsgen.sty 1999/11/30 v2.0 generic functions +\@emptytoks=\toks24 +\ex@=\dimen153 +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/amsmath/amsbsy.sty +Package: amsbsy 1999/11/29 v1.2d Bold Symbols +\pmbraise@=\dimen154 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/amsmath/amsopn.sty +Package: amsopn 2022/04/08 v2.04 operator names +) +\inf@bad=\count407 +LaTeX Info: Redefining \frac on input line 233. +\uproot@=\count408 +\leftroot@=\count409 +LaTeX Info: Redefining \overline on input line 398. +LaTeX Info: Redefining \colon on input line 409. +\classnum@=\count410 +\DOTSCASE@=\count411 +LaTeX Info: Redefining \ldots on input line 495. +LaTeX Info: Redefining \dots on input line 498. +LaTeX Info: Redefining \cdots on input line 619. +\Mathstrutbox@=\box55 +\strutbox@=\box56 +LaTeX Info: Redefining \big on input line 721. +LaTeX Info: Redefining \Big on input line 722. +LaTeX Info: Redefining \bigg on input line 723. +LaTeX Info: Redefining \Bigg on input line 724. +\big@size=\dimen155 +LaTeX Font Info: Redeclaring font encoding OML on input line 742. +LaTeX Font Info: Redeclaring font encoding OMS on input line 743. +\macc@depth=\count412 +LaTeX Info: Redefining \bmod on input line 904. +LaTeX Info: Redefining \pmod on input line 909. +LaTeX Info: Redefining \smash on input line 939. +LaTeX Info: Redefining \relbar on input line 969. +LaTeX Info: Redefining \Relbar on input line 970. +\c@MaxMatrixCols=\count413 +\dotsspace@=\muskip22 +\c@parentequation=\count414 +\dspbrk@lvl=\count415 +\tag@help=\toks25 +\row@=\count416 +\column@=\count417 +\maxfields@=\count418 +\andhelp@=\toks26 +\eqnshift@=\dimen156 +\alignsep@=\dimen157 +\tagshift@=\dimen158 +\tagwidth@=\dimen159 +\totwidth@=\dimen160 +\lineht@=\dimen161 +\@envbody=\toks27 +\multlinegap=\skip75 +\multlinetaggap=\skip76 +\mathdisplay@stack=\toks28 +LaTeX Info: Redefining \[ on input line 2953. +LaTeX Info: Redefining \] on input line 2954. +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/tabularray/tabularray.sty +Package: tabularray 2024-02-16 v2024A Typeset tabulars and arrays with LaTeX3 +\l__tblr_a_int=\count419 +\l__tblr_c_int=\count420 +\l__tblr_r_int=\count421 +\l__tblr_d_dim=\dimen162 +\l__tblr_h_dim=\dimen163 +\l__tblr_o_dim=\dimen164 +\l__tblr_p_dim=\dimen165 +\l__tblr_q_dim=\dimen166 +\l__tblr_r_dim=\dimen167 +\l__tblr_s_dim=\dimen168 +\l__tblr_t_dim=\dimen169 +\l__tblr_v_dim=\dimen170 +\l__tblr_w_dim=\dimen171 +\l__tblr_a_box=\box57 +\l__tblr_b_box=\box58 +\l__tblr_c_box=\box59 +\l__tblr_d_box=\box60 +\g__tblr_table_count_int=\count422 +\c@rownum=\count423 +\c@colnum=\count424 +\c@rowcount=\count425 +\c@colcount=\count426 +\abovesep=\dimen172 +\belowsep=\dimen173 +\leftsep=\dimen174 +\rightsep=\dimen175 +\g_tblr_level_int=\count427 +\g__tblr_data_row_key_count_int=\count428 +\g__tblr_data_column_key_count_int=\count429 +\g__tblr_data_cell_key_count_int=\count430 +\g__tblr_array_int=\count431 +\l__tblr_key_count_int=\count432 +\l__tblr_key_quotient_int=\count433 +\l__tblr_key_quotient_two_int=\count434 +\l__tblr_key_remainder_int=\count435 +\g__tblr_data_str_value_count_int=\count436 +\rulewidth=\dimen176 +\l__tblr_strut_dp_dim=\dimen177 +\l__tblr_strut_ht_dim=\dimen178 +\g__tblr_cell_wd_dim=\dimen179 +\g__tblr_cell_ht_dim=\dimen180 +\g__tblr_cell_head_dim=\dimen181 +\g__tblr_cell_foot_dim=\dimen182 +\l__column_target_dim=\dimen183 +\l__tblr_caption_box=\box61 +\l__tblr_caption_left_box=\box62 +\l__tblr_row_head_box=\box63 +\l__tblr_row_foot_box=\box64 +\l__tblr_row_head_foot_dim=\dimen184 +\tablewidth=\dimen185 +\l__tblr_table_firsthead_box=\box65 +\l__tblr_table_middlehead_box=\box66 +\l__tblr_table_lasthead_box=\box67 +\l__tblr_table_firstfoot_box=\box68 +\l__tblr_table_middlefoot_box=\box69 +\l__tblr_table_lastfoot_box=\box70 +\l__tblr_remain_height_dim=\dimen186 +\l__tblr_long_from_int=\count437 +\l__tblr_long_to_int=\count438 +\l__tblr_curr_i_int=\count439 +\l__tblr_prev_i_int=\count440 +\l__tblr_table_page_int=\count441 +\l__tblr_table_head_box=\box71 +\l__tblr_table_foot_box=\box72 +\l__tblr_table_head_foot_dim=\dimen187 +\l__tblr_table_head_body_foot_dim=\dimen188 +\l__tblr_table_box=\box73 +\l__tblr_table_hlines_box=\box74 +\l__tblr_hline_box=\box75 +\l__tblr_row_box=\box76 +\l__tblr_col_o_wd_dim=\dimen189 +\l__tblr_col_b_wd_dim=\dimen190 +\l__tblr_hline_leftskip_dim=\dimen191 +\l__tblr_hline_rightskip_dim=\dimen192 +\l__tblr_row_ht_dim=\dimen193 +\l__tblr_row_dp_dim=\dimen194 +\l__tblr_row_abovesep_dim=\dimen195 +\l__tblr_row_belowsep_dim=\dimen196 +\l__tblr_row_vlines_box=\box77 +\l__tblr_vline_box=\box78 +\l__tblr_cell_box=\box79 +\l__row_upper_dim=\dimen197 +\l__row_lower_dim=\dimen198 +\l__row_vpace_dim=\dimen199 +\l__tblr_vline_aboveskip_dim=\dimen256 +\l__tblr_vline_belowskip_dim=\dimen257 +\l__tblr_cell_wd_dim=\dimen258 +\l__tblr_cell_ht_dim=\dimen259 +\l__tblr_diag_box=\box80 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/booktabs/booktabs.sty +Package: booktabs 2020/01/12 v1.61803398 Publication quality tables +\heavyrulewidth=\dimen260 +\lightrulewidth=\dimen261 +\cmidrulewidth=\dimen262 +\belowrulesep=\dimen263 +\belowbottomsep=\dimen264 +\aboverulesep=\dimen265 +\abovetopsep=\dimen266 +\cmidrulesep=\dimen267 +\cmidrulekern=\dimen268 +\defaultaddspace=\dimen269 +\@cmidla=\count442 +\@cmidlb=\count443 +\@aboverulesep=\dimen270 +\@belowrulesep=\dimen271 +\@thisruleclass=\count444 +\@lastruleclass=\count445 +\@thisrulewidth=\dimen272 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/jknapltx/mathrsfs.sty +Package: mathrsfs 1996/01/01 Math RSFS package v1.0 (jk) +\symrsfs=\mathgroup6 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/mathtools/mathtools.sty +Package: mathtools 2024/03/11 v1.30 mathematical typesetting tools + +(/usr/local/texlive/2024/texmf-dist/tex/latex/tools/calc.sty +Package: calc 2023/07/08 v4.3 Infix arithmetic (KKT,FJ) +\calc@Acount=\count446 +\calc@Bcount=\count447 +\calc@Adimen=\dimen273 +\calc@Bdimen=\dimen274 +\calc@Askip=\skip77 +\calc@Bskip=\skip78 +LaTeX Info: Redefining \setlength on input line 80. +LaTeX Info: Redefining \addtolength on input line 81. +\calc@Ccount=\count448 +\calc@Cskip=\skip79 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/mathtools/mhsetup.sty +Package: mhsetup 2021/03/18 v1.4 programming setup (MH) +) +\g_MT_multlinerow_int=\count449 +\l_MT_multwidth_dim=\dimen275 +\origjot=\skip80 +\l_MT_shortvdotswithinadjustabove_dim=\dimen276 +\l_MT_shortvdotswithinadjustbelow_dim=\dimen277 +\l_MT_above_intertext_sep=\dimen278 +\l_MT_below_intertext_sep=\dimen279 +\l_MT_above_shortintertext_sep=\dimen280 +\l_MT_below_shortintertext_sep=\dimen281 +\xmathstrut@box=\box81 +\xmathstrut@dim=\dimen282 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/algorithmicx/algpseudocode.sty +Package: algpseudocode + +(/usr/local/texlive/2024/texmf-dist/tex/latex/algorithmicx/algorithmicx.sty +Package: algorithmicx 2005/04/27 v1.2 Algorithmicx + +Document Style algorithmicx 1.2 - a greatly improved `algorithmic' style +\c@ALG@line=\count450 +\c@ALG@rem=\count451 +\c@ALG@nested=\count452 +\ALG@tlm=\skip81 +\ALG@thistlm=\skip82 +\c@ALG@Lnr=\count453 +\c@ALG@blocknr=\count454 +\c@ALG@storecount=\count455 +\c@ALG@tmpcounter=\count456 +\ALG@tmplength=\skip83 +) +Document Style - pseudocode environments for use with the `algorithmicx' style +) (/usr/local/texlive/2024/texmf-dist/tex/latex/amscls/amsthm.sty +Package: amsthm 2020/05/29 v2.20.6 +\thm@style=\toks29 +\thm@bodyfont=\toks30 +\thm@headfont=\toks31 +\thm@notefont=\toks32 +\thm@headpunct=\toks33 +\thm@preskip=\skip84 +\thm@postskip=\skip85 +\thm@headsep=\skip86 +\dth@everypar=\toks34 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/thmtools/thmtools.sty +Package: thmtools 2023/05/04 v0.76 +\thmt@toks=\toks35 +\c@thmt@dummyctr=\count457 + +(/usr/local/texlive/2024/texmf-dist/tex/latex/thmtools/thm-patch.sty +Package: thm-patch 2023/05/04 v0.76 + +(/usr/local/texlive/2024/texmf-dist/tex/latex/thmtools/parseargs.sty +Package: parseargs 2023/05/04 v0.76 +\@parsespec=\toks36 +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/thmtools/thm-kv.sty +Package: thm-kv 2023/05/04 v0.76 +Package thm-kv Info: Theorem names will be uppercased on input line 42. +Package thm-kv Info: kvsetkeys patch (v1.16 or later) on input line 158. +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/thmtools/thm-autoref.sty +Package: thm-autoref 2023/05/04 v0.76 + +(/usr/local/texlive/2024/texmf-dist/tex/latex/thmtools/aliasctr.sty +Package: aliasctr 2023/05/04 v0.76 +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/thmtools/thm-listof.sty +Package: thm-listof 2023/05/04 v0.76 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/thmtools/thm-restate.sty +Package: thm-restate 2023/05/04 v0.76 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/thmtools/thm-amsthm.sty +Package: thm-amsthm 2023/05/04 v0.76 +\thmt@style@headstyle=\toks37 +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgfplots/pgfplots.sty +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplots.revision.tex) +Package: pgfplots 2021/05/15 v1.18.1 Data Visualization (1.18.1) +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgf/frontendlayer/tikz.sty +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgf/basiclayer/pgf.sty +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgf/utilities/pgfrcs.sty +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/utilities/pgfutil-common.te +x +\pgfutil@everybye=\toks38 +\pgfutil@tempdima=\dimen283 +\pgfutil@tempdimb=\dimen284 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/utilities/pgfutil-latex.def +\pgfutil@abb=\box82 +) (/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/utilities/pgfrcs.code.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/pgf.revision.tex) +Package: pgfrcs 2023-01-15 v3.1.10 (3.1.10) +)) +Package: pgf 2023-01-15 v3.1.10 (3.1.10) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgf/basiclayer/pgfcore.sty +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgf/systemlayer/pgfsys.sty +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/systemlayer/pgfsys.code.tex +Package: pgfsys 2023-01-15 v3.1.10 (3.1.10) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex +\pgfkeys@pathtoks=\toks39 +\pgfkeys@temptoks=\toks40 + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/utilities/pgfkeyslibraryfil +tered.code.tex +\pgfkeys@tmptoks=\toks41 +)) +\pgf@x=\dimen285 +\pgf@y=\dimen286 +\pgf@xa=\dimen287 +\pgf@ya=\dimen288 +\pgf@xb=\dimen289 +\pgf@yb=\dimen290 +\pgf@xc=\dimen291 +\pgf@yc=\dimen292 +\pgf@xd=\dimen293 +\pgf@yd=\dimen294 +\w@pgf@writea=\write4 +\r@pgf@reada=\read4 +\c@pgf@counta=\count458 +\c@pgf@countb=\count459 +\c@pgf@countc=\count460 +\c@pgf@countd=\count461 +\t@pgf@toka=\toks42 +\t@pgf@tokb=\toks43 +\t@pgf@tokc=\toks44 +\pgf@sys@id@count=\count462 + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/systemlayer/pgf.cfg +File: pgf.cfg 2023-01-15 v3.1.10 (3.1.10) +) +Driver file for pgf: pgfsys-pdftex.def + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-pdftex.d +ef +File: pgfsys-pdftex.def 2023-01-15 v3.1.10 (3.1.10) + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-common-p +df.def +File: pgfsys-common-pdf.def 2023-01-15 v3.1.10 (3.1.10) +))) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/systemlayer/pgfsyssoftpath. +code.tex +File: pgfsyssoftpath.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgfsyssoftpath@smallbuffer@items=\count463 +\pgfsyssoftpath@bigbuffer@items=\count464 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/systemlayer/pgfsysprotocol. +code.tex +File: pgfsysprotocol.code.tex 2023-01-15 v3.1.10 (3.1.10) +)) (/usr/local/texlive/2024/texmf-dist/tex/latex/xcolor/xcolor.sty +Package: xcolor 2023/11/15 v3.01 LaTeX color extensions (UK) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/graphics-cfg/color.cfg +File: color.cfg 2016/01/02 v1.6 sample color configuration +) +Package xcolor Info: Driver file: pdftex.def on input line 274. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/graphics/mathcolor.ltx) +Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1350. +Package xcolor Info: Model `hsb' substituted by `rgb' on input line 1354. +Package xcolor Info: Model `RGB' extended on input line 1366. +Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1368. +Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1369. +Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1370. +Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1371. +Package xcolor Info: Model `Gray' substituted by `gray' on input line 1372. +Package xcolor Info: Model `wave' substituted by `hsb' on input line 1373. +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcore.code.tex +Package: pgfcore 2023-01-15 v3.1.10 (3.1.10) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathutil.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathparser.code.tex +\pgfmath@dimen=\dimen295 +\pgfmath@count=\count465 +\pgfmath@box=\box83 +\pgfmath@toks=\toks45 +\pgfmath@stack@operand=\toks46 +\pgfmath@stack@operation=\toks47 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.code. +tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.basic +.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.trigo +nometric.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.rando +m.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.compa +rison.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.base. +code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.round +.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.misc. +code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.integ +erarithmetics.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathcalc.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmathfloat.code.tex +\c@pgfmathroundto@lastzeros=\count466 +)) (/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfint.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepoints.co +de.tex +File: pgfcorepoints.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgf@picminx=\dimen296 +\pgf@picmaxx=\dimen297 +\pgf@picminy=\dimen298 +\pgf@picmaxy=\dimen299 +\pgf@pathminx=\dimen300 +\pgf@pathmaxx=\dimen301 +\pgf@pathminy=\dimen302 +\pgf@pathmaxy=\dimen303 +\pgf@xx=\dimen304 +\pgf@xy=\dimen305 +\pgf@yx=\dimen306 +\pgf@yy=\dimen307 +\pgf@zx=\dimen308 +\pgf@zy=\dimen309 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathconst +ruct.code.tex +File: pgfcorepathconstruct.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgf@path@lastx=\dimen310 +\pgf@path@lasty=\dimen311 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathusage +.code.tex +File: pgfcorepathusage.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgf@shorten@end@additional=\dimen312 +\pgf@shorten@start@additional=\dimen313 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcorescopes.co +de.tex +File: pgfcorescopes.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgfpic=\box84 +\pgf@hbox=\box85 +\pgf@layerbox@main=\box86 +\pgf@picture@serial@count=\count467 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcoregraphicst +ate.code.tex +File: pgfcoregraphicstate.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgflinewidth=\dimen314 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransform +ations.code.tex +File: pgfcoretransformations.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgf@pt@x=\dimen315 +\pgf@pt@y=\dimen316 +\pgf@pt@temp=\dimen317 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcorequick.cod +e.tex +File: pgfcorequick.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreobjects.c +ode.tex +File: pgfcoreobjects.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathproce +ssing.code.tex +File: pgfcorepathprocessing.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcorearrows.co +de.tex +File: pgfcorearrows.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgfarrowsep=\dimen318 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreshade.cod +e.tex +File: pgfcoreshade.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgf@max=\dimen319 +\pgf@sys@shading@range@num=\count468 +\pgf@shadingcount=\count469 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreimage.cod +e.tex +File: pgfcoreimage.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreexternal. +code.tex +File: pgfcoreexternal.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgfexternal@startupbox=\box87 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcorelayers.co +de.tex +File: pgfcorelayers.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretranspare +ncy.code.tex +File: pgfcoretransparency.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepatterns. +code.tex +File: pgfcorepatterns.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/basiclayer/pgfcorerdf.code. +tex +File: pgfcorerdf.code.tex 2023-01-15 v3.1.10 (3.1.10) +))) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/modules/pgfmoduleshapes.cod +e.tex +File: pgfmoduleshapes.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgfnodeparttextbox=\box88 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/modules/pgfmoduleplot.code. +tex +File: pgfmoduleplot.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version +-0-65.sty +Package: pgfcomp-version-0-65 2023-01-15 v3.1.10 (3.1.10) +\pgf@nodesepstart=\dimen320 +\pgf@nodesepend=\dimen321 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version +-1-18.sty +Package: pgfcomp-version-1-18 2023-01-15 v3.1.10 (3.1.10) +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgf/utilities/pgffor.sty +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgf/utilities/pgfkeys.sty +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex) +) (/usr/local/texlive/2024/texmf-dist/tex/latex/pgf/math/pgfmath.sty +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex)) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/utilities/pgffor.code.tex +Package: pgffor 2023-01-15 v3.1.10 (3.1.10) +\pgffor@iter=\dimen322 +\pgffor@skip=\dimen323 +\pgffor@stack=\toks48 +\pgffor@toks=\toks49 +)) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/frontendlayer/tikz/tikz.cod +e.tex +Package: tikz 2023-01-15 v3.1.10 (3.1.10) + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/libraries/pgflibraryplothan +dlers.code.tex +File: pgflibraryplothandlers.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgf@plot@mark@count=\count470 +\pgfplotmarksize=\dimen324 +) +\tikz@lastx=\dimen325 +\tikz@lasty=\dimen326 +\tikz@lastxsaved=\dimen327 +\tikz@lastysaved=\dimen328 +\tikz@lastmovetox=\dimen329 +\tikz@lastmovetoy=\dimen330 +\tikzleveldistance=\dimen331 +\tikzsiblingdistance=\dimen332 +\tikz@figbox=\box89 +\tikz@figbox@bg=\box90 +\tikz@tempbox=\box91 +\tikz@tempbox@bg=\box92 +\tikztreelevel=\count471 +\tikznumberofchildren=\count472 +\tikznumberofcurrentchild=\count473 +\tikz@fig@count=\count474 + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/modules/pgfmodulematrix.cod +e.tex +File: pgfmodulematrix.code.tex 2023-01-15 v3.1.10 (3.1.10) +\pgfmatrixcurrentrow=\count475 +\pgfmatrixcurrentcolumn=\count476 +\pgf@matrix@numberofcolumns=\count477 +) +\tikz@expandcount=\count478 + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibrarytopaths.code.tex +File: tikzlibrarytopaths.code.tex 2023-01-15 v3.1.10 (3.1.10) +))) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplots.code.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplotscore.code.tex +\t@pgfplots@toka=\toks50 +\t@pgfplots@tokb=\toks51 +\t@pgfplots@tokc=\toks52 +\pgfplots@tmpa=\dimen333 +\c@pgfplots@coordindex=\count479 +\c@pgfplots@scanlineindex=\count480 + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/sys/pgfplotssysgeneric +.code.tex)) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/libs/pgfplotslibrary.c +ode.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/oldpgfcompatib/pgfplot +soldpgfsupp_loader.code.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/libraries/pgflibraryfpu.cod +e.tex) +Package pgfplots: loading complementary utilities for your pgf version... +\t@pgf@toka=\toks53 +\t@pgf@tokb=\toks54 +\t@pgf@tokc=\toks55 + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/oldpgfcompatib/pgfplot +soldpgfsupp_pgfutil-common-lists.tex)) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/util/pgfplotsutil.code +.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/liststructure/pgfplots +liststructure.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/liststructure/pgfplots +liststructureext.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/liststructure/pgfplots +array.code.tex +\c@pgfplotsarray@tmp=\count481 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/liststructure/pgfplots +matrix.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/numtable/pgfplotstable +shared.code.tex +\c@pgfplotstable@counta=\count482 +\t@pgfplotstable@a=\toks56 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/liststructure/pgfplots +deque.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/util/pgfplotsbinary.co +de.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/util/pgfplotsbinary.da +ta.code.tex)) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/util/pgfplotsutil.verb +.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/libs/pgflibrarypgfplot +s.surfshading.code.tex +\c@pgfplotslibrarysurf@no=\count483 + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/sys/pgflibrarypgfplots +.surfshading.pgfsys-pdftex.def))) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/util/pgfplotscolormap. +code.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/util/pgfplotscolor.cod +e.tex)) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplotsstackedplots.c +ode.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplotsplothandlers.c +ode.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplotsmeshplothandle +r.code.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplotsmeshplotimage. +code.tex))) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplots.scaling.code. +tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplotscoordprocessin +g.code.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplots.errorbars.cod +e.tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplots.markers.code. +tex) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplotsticks.code.tex +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/pgfplots.paths.code.te +x) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibrarydecorations.code.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/modules/pgfmoduledecoration +s.code.tex +\pgfdecoratedcompleteddistance=\dimen334 +\pgfdecoratedremainingdistance=\dimen335 +\pgfdecoratedinputsegmentcompleteddistance=\dimen336 +\pgfdecoratedinputsegmentremainingdistance=\dimen337 +\pgf@decorate@distancetomove=\dimen338 +\pgf@decorate@repeatstate=\count484 +\pgfdecorationsegmentamplitude=\dimen339 +\pgfdecorationsegmentlength=\dimen340 +) +\tikz@lib@dec@box=\box93 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibrarydecorations.pathmorphing.code.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/libraries/decorations/pgfli +brarydecorations.pathmorphing.code.tex)) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibrarydecorations.pathreplacing.code.tex +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/libraries/decorations/pgfli +brarydecorations.pathreplacing.code.tex)) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgfplots/libs/tikzlibrarypgfplo +ts.contourlua.code.tex) +\pgfplots@numplots=\count485 +\pgfplots@xmin@reg=\dimen341 +\pgfplots@xmax@reg=\dimen342 +\pgfplots@ymin@reg=\dimen343 +\pgfplots@ymax@reg=\dimen344 +\pgfplots@zmin@reg=\dimen345 +\pgfplots@zmax@reg=\dimen346 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibraryplotmarks.code.tex +File: tikzlibraryplotmarks.code.tex 2023-01-15 v3.1.10 (3.1.10) + +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/libraries/pgflibraryplotmar +ks.code.tex +File: pgflibraryplotmarks.code.tex 2023-01-15 v3.1.10 (3.1.10) +))) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibrarytrees.code.tex +File: tikzlibrarytrees.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibrarymatrix.code.tex +File: tikzlibrarymatrix.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibrarypositioning.code.tex +File: tikzlibrarypositioning.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibraryfit.code.tex +File: tikzlibraryfit.code.tex 2023-01-15 v3.1.10 (3.1.10) +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/enumitem/enumitem.sty +Package: enumitem 2019/06/20 v3.9 Customized lists +\labelindent=\skip87 +\enit@outerparindent=\dimen347 +\enit@toks=\toks57 +\enit@inbox=\box94 +\enit@count@id=\count486 +\enitdp@description=\count487 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/colortbl/colortbl.sty +Package: colortbl 2024/05/26 v1.0h Color table columns (DPC) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/tools/array.sty +Package: array 2024/06/14 v2.6d Tabular extension package (FMi) +\col@sep=\dimen348 +\ar@mcellbox=\box95 +\extrarowheight=\dimen349 +\NC@list=\toks58 +\extratabsurround=\skip88 +\backup@length=\skip89 +\ar@cellbox=\box96 +) +\everycr=\toks59 +\minrowclearance=\skip90 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/makecell/makecell.sty +Package: makecell 2009/08/03 V0.1e Managing of Tab Column Heads and Cells +\rotheadsize=\dimen350 +\c@nlinenum=\count488 +\TeXr@lab=\toks60 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/cancel/cancel.sty +Package: cancel 2013/04/12 v2.2 Cancel math terms +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgfornament/pgfornament.sty +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgfopts/pgfopts.sty +Package: pgfopts 2014/07/10 v2.1a LaTeX package options with pgfkeys +\pgfopts@list@add@a@toks=\toks61 +\pgfopts@list@add@b@toks=\toks62 +) +Package: pgfornament 2020/05/26 v1.2 vector ornaments + +(/usr/local/texlive/2024/texmf-dist/tex/latex/pgfornament/pgflibraryvectorian.c +ode.tex +2020/04/06 1.1 pgflibraryvectorian.code.tex +)) (/usr/local/texlive/2024/texmf-dist/tex/latex/tools/multicol.sty +Package: multicol 2024/05/23 v1.9h multicolumn formatting (FMi) +\c@tracingmulticols=\count489 +\mult@box=\box97 +\multicol@leftmargin=\dimen351 +\c@unbalance=\count490 +\c@collectmore=\count491 +\doublecol@number=\count492 +\multicoltolerance=\count493 +\multicolpretolerance=\count494 +\full@width=\dimen352 +\page@free=\dimen353 +\premulticols=\dimen354 +\postmulticols=\dimen355 +\multicolsep=\skip91 +\multicolbaselineskip=\skip92 +\partial@page=\box98 +\last@line=\box99 +\mc@boxedresult=\box100 +\maxbalancingoverflow=\dimen356 +\mult@rightbox=\box101 +\mult@grightbox=\box102 +\mult@firstbox=\box103 +\mult@gfirstbox=\box104 +\@tempa=\box105 +\@tempa=\box106 +\@tempa=\box107 +\@tempa=\box108 +\@tempa=\box109 +\@tempa=\box110 +\@tempa=\box111 +\@tempa=\box112 +\@tempa=\box113 +\@tempa=\box114 +\@tempa=\box115 +\@tempa=\box116 +\@tempa=\box117 +\@tempa=\box118 +\@tempa=\box119 +\@tempa=\box120 +\@tempa=\box121 +\@tempa=\box122 +\@tempa=\box123 +\@tempa=\box124 +\@tempa=\box125 +\@tempa=\box126 +\@tempa=\box127 +\@tempa=\box128 +\@tempa=\box129 +\@tempa=\box130 +\@tempa=\box131 +\@tempa=\box132 +\@tempa=\box133 +\@tempa=\box134 +\@tempa=\box135 +\@tempa=\box136 +\@tempa=\box137 +\@tempa=\box138 +\@tempa=\box139 +\@tempa=\box140 +\c@minrows=\count495 +\c@columnbadness=\count496 +\c@finalcolumnbadness=\count497 +\last@try=\dimen357 +\multicolovershoot=\dimen358 +\multicolundershoot=\dimen359 +\mult@nat@firstbox=\box141 +\colbreak@box=\box142 +\mc@col@check@num=\count498 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/caption/subcaption.sty +Package: subcaption 2023/07/28 v1.6b Sub-captions (AR) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/caption/caption.sty +Package: caption 2023/08/05 v3.6o Customizing captions (AR) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/caption/caption3.sty +Package: caption3 2023/07/31 v2.4d caption3 kernel (AR) +\caption@tempdima=\dimen360 +\captionmargin=\dimen361 +\caption@leftmargin=\dimen362 +\caption@rightmargin=\dimen363 +\caption@width=\dimen364 +\caption@indent=\dimen365 +\caption@parindent=\dimen366 +\caption@hangindent=\dimen367 +Package caption Info: Standard document class detected. +) +\c@caption@flags=\count499 +\c@continuedfloat=\count500 +Package caption Info: float package is loaded. +) +Package caption Info: New subtype `subfigure' on input line 238. +\c@subfigure=\count501 +Package caption Info: New subtype `subtable' on input line 238. +\c@subtable=\count502 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/bold-extra/bold-extra.sty +Package: bold-extra 2001/11/13 v0.1 Use fonts from cm/mf-extra/bold +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/csquotes/csquotes.sty +Package: csquotes 2024-04-04 v5.2o context-sensitive quotations (JAW) +\csq@reset=\count503 +\csq@gtype=\count504 +\csq@glevel=\count505 +\csq@qlevel=\count506 +\csq@maxlvl=\count507 +\csq@tshold=\count508 +\csq@ltx@everypar=\toks63 + +(/usr/local/texlive/2024/texmf-dist/tex/latex/csquotes/csquotes.def +File: csquotes.def 2024-04-04 v5.2o csquotes generic definitions (JAW) +) +Package csquotes Info: Trying to load configuration file 'csquotes.cfg'... +Package csquotes Info: ... configuration file loaded successfully. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/csquotes/csquotes.cfg +File: csquotes.cfg +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/hyperref/hyperref.sty +Package: hyperref 2024-05-23 v7.01i Hypertext links for LaTeX + +(/usr/local/texlive/2024/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty +Package: kvdefinekeys 2019-12-19 v1.6 Define keys (HO) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/pdfescape/pdfescape.sty +Package: pdfescape 2019/12/09 v1.15 Implements pdfTeX's escape features (HO) +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/hycolor/hycolor.sty +Package: hycolor 2020-01-27 v1.10 Color options for hyperref/bookmark (HO) +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/hyperref/nameref.sty +Package: nameref 2023-11-26 v2.56 Cross-referencing by name of section + +(/usr/local/texlive/2024/texmf-dist/tex/latex/refcount/refcount.sty +Package: refcount 2019/12/15 v3.6 Data extraction from label references (HO) +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/gettitlestring/gettitlestring.s +ty +Package: gettitlestring 2019/12/15 v1.6 Cleanup title references (HO) +) +\c@section@level=\count509 +) (/usr/local/texlive/2024/texmf-dist/tex/generic/stringenc/stringenc.sty +Package: stringenc 2019/11/29 v1.12 Convert strings between diff. encodings (HO +) +) +\@linkdim=\dimen368 +\Hy@linkcounter=\count510 +\Hy@pagecounter=\count511 + +(/usr/local/texlive/2024/texmf-dist/tex/latex/hyperref/pd1enc.def +File: pd1enc.def 2024-05-23 v7.01i Hyperref: PDFDocEncoding definition (HO) +Now handling font encoding PD1 ... +... no UTF-8 mapping file for font encoding PD1 +) +(/usr/local/texlive/2024/texmf-dist/tex/generic/intcalc/intcalc.sty +Package: intcalc 2019/12/15 v1.3 Expandable calculations with integers (HO) +) +\Hy@SavedSpaceFactor=\count512 + +(/usr/local/texlive/2024/texmf-dist/tex/latex/hyperref/puenc.def +File: puenc.def 2024-05-23 v7.01i Hyperref: PDF Unicode definition (HO) +Now handling font encoding PU ... +... no UTF-8 mapping file for font encoding PU +) +Package hyperref Info: Hyper figures OFF on input line 4157. +Package hyperref Info: Link nesting OFF on input line 4162. +Package hyperref Info: Hyper index ON on input line 4165. +Package hyperref Info: Plain pages OFF on input line 4172. +Package hyperref Info: Backreferencing OFF on input line 4177. +Package hyperref Info: Implicit mode ON; LaTeX internals redefined. +Package hyperref Info: Bookmarks ON on input line 4424. +\c@Hy@tempcnt=\count513 +LaTeX Info: Redefining \url on input line 4763. +\XeTeXLinkMargin=\dimen369 + +(/usr/local/texlive/2024/texmf-dist/tex/generic/bitset/bitset.sty +Package: bitset 2019/12/09 v1.3 Handle bit-vector datatype (HO) + +(/usr/local/texlive/2024/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty +Package: bigintcalc 2019/12/15 v1.5 Expandable calculations on big integers (HO +) +)) +\Fld@menulength=\count514 +\Field@Width=\dimen370 +\Fld@charsize=\dimen371 +Package hyperref Info: Hyper figures OFF on input line 6042. +Package hyperref Info: Link nesting OFF on input line 6047. +Package hyperref Info: Hyper index ON on input line 6050. +Package hyperref Info: backreferencing OFF on input line 6057. +Package hyperref Info: Link coloring OFF on input line 6062. +Package hyperref Info: Link coloring with OCG OFF on input line 6067. +Package hyperref Info: PDF/A mode OFF on input line 6072. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/base/atbegshi-ltx.sty +Package: atbegshi-ltx 2021/01/10 v1.0c Emulation of the original atbegshi +package with kernel methods +) +\Hy@abspage=\count515 +\c@Item=\count516 +\c@Hfootnote=\count517 +) +Package hyperref Info: Driver (autodetected): hpdftex. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/hyperref/hpdftex.def +File: hpdftex.def 2024-05-23 v7.01i Hyperref driver for pdfTeX + +(/usr/local/texlive/2024/texmf-dist/tex/latex/base/atveryend-ltx.sty +Package: atveryend-ltx 2020/08/19 v1.0a Emulation of the original atveryend pac +kage +with kernel methods +) +\Fld@listcount=\count518 +\c@bookmark@seq@number=\count519 + +(/usr/local/texlive/2024/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty +Package: rerunfilecheck 2022-07-10 v1.10 Rerun checks for auxiliary files (HO) + +(/usr/local/texlive/2024/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty +Package: uniquecounter 2019/12/15 v1.4 Provide unlimited unique counter (HO) +) +Package uniquecounter Info: New unique counter `rerunfilecheck' on input line 2 +85. +) +\Hy@SectionHShift=\skip93 +) (/usr/local/texlive/2024/texmf-dist/tex/latex/bookmark/bookmark.sty +Package: bookmark 2023-12-10 v1.31 PDF bookmarks (HO) + +(/usr/local/texlive/2024/texmf-dist/tex/latex/bookmark/bkm-pdftex.def +File: bkm-pdftex.def 2023-12-10 v1.31 bookmark driver for pdfTeX and luaTeX (HO +) +\BKM@id=\count520 +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/algorithm2e/algorithm2e.sty +Package: algorithm2e 2017/07/18 v5.2 algorithms environments +\c@AlgoLine=\count521 +\algocf@hangindent=\skip94 + +(/usr/local/texlive/2024/texmf-dist/tex/latex/ifoddpage/ifoddpage.sty +Package: ifoddpage 2022/10/18 v1.2 Conditionals for odd/even page detection +\c@checkoddpage=\count522 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/tools/xspace.sty +Package: xspace 2014/10/28 v1.13 Space after command names (DPC,MH) +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/relsize/relsize.sty +Package: relsize 2013/03/29 ver 4.1 +) +\skiptotal=\skip95 +\skiplinenumber=\skip96 +\skiprule=\skip97 +\skiphlne=\skip98 +\skiptext=\skip99 +\skiplength=\skip100 +\algomargin=\skip101 +\skipalgocfslide=\skip102 +\algowidth=\dimen372 +\inoutsize=\dimen373 +\inoutindent=\dimen374 +\interspacetitleruled=\dimen375 +\interspacealgoruled=\dimen376 +\interspacetitleboxruled=\dimen377 +\algocf@ruledwidth=\skip103 +\algocf@inoutbox=\box143 +\algocf@inputbox=\box144 +\AlCapSkip=\skip104 +\AlCapHSkip=\skip105 +\algoskipindent=\skip106 +\algocf@nlbox=\box145 +\algocf@hangingbox=\box146 +\algocf@untilbox=\box147 +\algocf@skipuntil=\skip107 +\algocf@capbox=\box148 +\algocf@lcaptionbox=\skip108 +\algoheightruledefault=\skip109 +\algoheightrule=\skip110 +\algotitleheightruledefault=\skip111 +\algotitleheightrule=\skip112 +\c@algocfline=\count523 +\c@algocfproc=\count524 +\c@algocf=\count525 +\algocf@algoframe=\box149 +\algocf@algobox=\box150 +) +Package thmtools Info: Key `qed' (with value `\qedsymbol {}') +(thmtools) is not a known style key. +(thmtools) Will pass this to every \declaretheorem +(thmtools) that uses `style=steo' on input line 154. +Package thmtools Info: Key `qed' (with value `\qedsymbol {}') +(thmtools) is not a known style key. +(thmtools) Will pass this to every \declaretheorem +(thmtools) that uses `style=slemma' on input line 162. +\c@mlemma=\count526 +Package thmtools Info: Key `qed' (with value `$\blacksquare $') +(thmtools) is not a known style key. +(thmtools) Will pass this to every \declaretheorem +(thmtools) that uses `style=sprf' on input line 180. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/titlesec/titlesec.sty +Package: titlesec 2023/10/27 v2.16 Sectioning titles +\ttl@box=\box151 +\beforetitleunit=\skip113 +\aftertitleunit=\skip114 +\ttl@plus=\dimen378 +\ttl@minus=\dimen379 +\ttl@toksa=\toks64 +\titlewidth=\dimen380 +\titlewidthlast=\dimen381 +\titlewidthfirst=\dimen382 +) +(/usr/local/texlive/2024/texmf-dist/tex/latex/setspace/setspace.sty +Package: setspace 2022/12/04 v6.7b set line spacing +) +Package csquotes Info: Checking for multilingual support... +Package csquotes Info: ... found 'babel' package. +Package csquotes Info: Adjusting default style. +Package csquotes Info: Redefining alias 'default' -> 'english'. + +(./main.aux (./(1) - introduction/introduction.aux) +(./(2) - problem definition/problem definition.aux) +(./(3) - algorithms/algorithms.aux) (./(4) - experiments/experiments.aux) +(./(6) - conclusion/conclusion.aux) (./(7) - proofs/proofs.aux)) +\openout1 = `main.aux'. + +LaTeX Font Info: Checking defaults for OML/cmm/m/it on input line 236. +LaTeX Font Info: ... okay on input line 236. +LaTeX Font Info: Checking defaults for OMS/cmsy/m/n on input line 236. +LaTeX Font Info: ... okay on input line 236. +LaTeX Font Info: Checking defaults for OT1/cmr/m/n on input line 236. +LaTeX Font Info: ... okay on input line 236. +LaTeX Font Info: Checking defaults for T1/cmr/m/n on input line 236. +LaTeX Font Info: ... okay on input line 236. +LaTeX Font Info: Checking defaults for TS1/cmr/m/n on input line 236. +LaTeX Font Info: ... okay on input line 236. +LaTeX Font Info: Checking defaults for OMX/cmex/m/n on input line 236. +LaTeX Font Info: ... okay on input line 236. +LaTeX Font Info: Checking defaults for U/cmr/m/n on input line 236. +LaTeX Font Info: ... okay on input line 236. +LaTeX Font Info: Checking defaults for PD1/pdf/m/n on input line 236. +LaTeX Font Info: ... okay on input line 236. +LaTeX Font Info: Checking defaults for PU/pdf/m/n on input line 236. +LaTeX Font Info: ... okay on input line 236. + +*geometry* driver: auto-detecting +*geometry* detected driver: pdftex +*geometry* verbose mode - [ preamble ] result: +* driver: pdftex +* paper: a4paper +* layout: +* layoutoffset:(h,v)=(0.0pt,0.0pt) +* hratio: 1:1 +* vratio: 1:1 +* modes: +* h-part:(L,W,R)=(89.62709pt, 418.25368pt, 89.6271pt) +* v-part:(T,H,B)=(126.75832pt, 591.5302pt, 126.75833pt) +* \paperwidth=597.50787pt +* \paperheight=845.04684pt +* \textwidth=418.25368pt +* \textheight=591.5302pt +* \oddsidemargin=17.3571pt +* \evensidemargin=17.3571pt +* \topmargin=9.48833pt +* \headheight=20.0pt +* \headsep=25.0pt +* \topskip=12.0pt +* \footskip=30.0pt +* \marginparwidth=44.0pt +* \marginparsep=10.0pt +* \columnsep=10.0pt +* \skip\footins=10.8pt plus 4.0pt minus 2.0pt +* \hoffset=0.0pt +* \voffset=0.0pt +* \mag=1000 +* \@twocolumnfalse +* \@twosidefalse +* \@mparswitchfalse +* \@reversemarginfalse +* (1in=72.27pt=25.4mm, 1cm=28.453pt) + +Package biblatex Info: Trying to load language 'english'... +Package biblatex Info: ... file 'english.lbx' found. +(/usr/local/texlive/2024/texmf-dist/tex/latex/biblatex/lbx/english.lbx +File: english.lbx 2024/03/21 v3.20 biblatex localization (PK/MW) +) +Package biblatex Info: Input encoding 'utf8' detected. +Package biblatex Info: Automatic encoding selection. +(biblatex) Assuming data encoding 'utf8'. +\openout3 = `main.bcf'. + +Package biblatex Info: Trying to load bibliographic data... +Package biblatex Info: ... file 'main.bbl' found. + +(./main.bbl) +Package biblatex Info: Reference section=0 on input line 236. +Package biblatex Info: Reference segment=0 on input line 236. + +(/usr/local/texlive/2024/texmf-dist/tex/context/base/mkii/supp-pdf.mkii +[Loading MPS to PDF converter (version 2006.09.02).] +\scratchcounter=\count527 +\scratchdimen=\dimen383 +\scratchbox=\box152 +\nofMPsegments=\count528 +\nofMParguments=\count529 +\everyMPshowfont=\toks65 +\MPscratchCnt=\count530 +\MPscratchDim=\dimen384 +\MPnumerator=\count531 +\makeMPintoPDFobject=\count532 +\everyMPtoPDFconversion=\toks66 +) (/usr/local/texlive/2024/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty +Package: epstopdf-base 2020-01-24 v2.11 Base part for package epstopdf +Package epstopdf-base Info: Redefining graphics rule for `.eps' on input line 4 +85. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg +File: epstopdf-sys.cfg 2010/07/13 v1.3 Configuration of (r)epstopdf for TeX Liv +e +)) +(/usr/local/texlive/2024/texmf-dist/tex/latex/ninecolors/ninecolors.sty +Package: ninecolors 2022-02-13 v2022D Select colors with proper color contrast +) +Package pgfplots notification 'compat/show suggested version=true': document ha +s been generated with the most recent feature set (\pgfplotsset{compat=1.18}). + +Package caption Info: Begin \AtBeginDocument code. +Package caption Info: hyperref package is loaded. +Package caption Info: End \AtBeginDocument code. +Package hyperref Info: Link coloring OFF on input line 236. + +File: images/UniPi Logo.png Graphic file (type png) + +Package pdftex.def Info: images/UniPi Logo.png used on input line 241. +(pdftex.def) Requested size: 111.65913pt x 113.81102pt. +LaTeX Font Info: Font shape `OT1/cmr/bx/sc' in size <24.88> not available +(Font) Font shape `OT1/cmr/b/sc' tried instead on input line 246. +LaTeX Font Info: Font shape `OT1/cmr/bx/sc' in size <20.74> not available +(Font) Font shape `OT1/cmr/b/sc' tried instead on input line 253. +> Step: init table outer spec. +> Step: parse table options. +> Step: split table. +> Step: init table inner spec. +> Step: parse table inner spec. +> Step: execute table commands. +> Step: calculate cell and line sizes. +> Step: init table outer spec. +> Step: parse table options. +> Step: split table. +> Step: init table inner spec. +> Step: parse table inner spec. +> Step: execute table commands. +> Step: calculate cell and line sizes. +> Step: build the whole table. +LaTeX Font Info: Trying to load font information for U+msa on input line 257 +. +(/usr/local/texlive/2024/texmf-dist/tex/latex/amsfonts/umsa.fd +File: umsa.fd 2013/01/14 v3.01 AMS symbols A +) +LaTeX Font Info: Trying to load font information for U+msb on input line 257 +. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/amsfonts/umsb.fd +File: umsb.fd 2013/01/14 v3.01 AMS symbols B +) +LaTeX Font Info: Trying to load font information for U+rsfs on input line 25 +7. + +(/usr/local/texlive/2024/texmf-dist/tex/latex/jknapltx/ursfs.fd +File: ursfs.fd 1998/03/24 rsfs font definition file (jk) +) +> Step: init table outer spec. +> Step: parse table options. +> Step: split table. +> Step: init table inner spec. +> Step: parse table inner spec. +> Step: execute table commands. +> Step: calculate cell and line sizes. +> Step: build the whole table. +> Step: init table outer spec. +> Step: parse table options. +> Step: split table. +> Step: init table inner spec. +> Step: parse table inner spec. +> Step: execute table commands. +> Step: calculate cell and line sizes. +> Step: build the whole table. +> Step: init table outer spec. +> Step: parse table options. +> Step: split table. +> Step: init table inner spec. +> Step: parse table inner spec. +> Step: execute table commands. +> Step: calculate cell and line sizes. +> Step: build the whole table. +> Step: build the whole table. +> Step: init table outer spec. +> Step: parse table options. +> Step: split table. +> Step: init table inner spec. +> Step: parse table inner spec. +> Step: execute table commands. +> Step: calculate cell and line sizes. +> Step: build the whole table. +> Step: init table outer spec. +> Step: parse table options. +> Step: split table. +> Step: init table inner spec. +> Step: parse table inner spec. +> Step: execute table commands. +> Step: calculate cell and line sizes. +> Step: build the whole table. +> Step: init table outer spec. +> Step: parse table options. +> Step: split table. +> Step: init table inner spec. +> Step: parse table inner spec. +> Step: execute table commands. +> Step: calculate cell and line sizes. +> Step: build the whole table. +> Step: init table outer spec. +> Step: parse table options. +> Step: split table. +> Step: init table inner spec. +> Step: parse table inner spec. +> Step: execute table commands. +> Step: calculate cell and line sizes. +> Step: build the whole table. + + +[1 + +{/usr/local/texlive/2024/texmf-var/fonts/map/pdftex/updmap/pdftex.map} <./image +s/UniPi Logo.png>] (./main.toc) +\tf@toc=\write5 +\openout5 = `main.toc'. + + + +[1 + +] +\openout2 = `"(1) - introduction/introduction.aux"'. + + (./(1) - introduction/introduction.tex +Chapter 1. +) + + +pdfTeX warning (ext4): destination with the same identifier (name{page.1}) has +been already used, duplicate ignored + + \relax +l.276 ...\include{(1) - introduction/introduction} + [1 + + + +] +\openout2 = `"(2) - problem definition/problem definition.aux"'. + + +(./(2) - problem definition/problem definition.tex +Chapter 2. + + +[2 + + + + +] +> Step: init table outer spec. +> Step: parse table options. +> Step: split table. +> Step: init table inner spec. +> Step: parse table inner spec. +> Step: execute table commands. +> Step: calculate cell and line sizes. +> Step: build the whole table. + + +[3] +> Step: init table outer spec. +> Step: parse table options. +> Step: split table. +> Step: init table inner spec. +> Step: parse table inner spec. +> Step: execute table commands. +> Step: calculate cell and line sizes. +> Step: build the whole table. + + +[4] +<(2) - problem definition/images/conditioning.png, id=96, 702.4644pt x 550.6974 +pt> +File: (2) - problem definition/images/conditioning.png Graphic file (type png) + +Package pdftex.def Info: (2) - problem definition/images/conditioning.png used + on input line 139. +(pdftex.def) Requested size: 292.77629pt x 229.5267pt. +) + +[5 <./(2) - problem definition/images/conditioning.png>] +\openout2 = `"(3) - algorithms/algorithms.aux"'. + + +(./(3) - algorithms/algorithms.tex +Chapter 3. + + +[6 + + + + +] + +[7] + +[8] + +[9] +Package hyperref Info: bookmark level for unknown mtheo defaults to 0 on input +line 180. +Package hyperref Info: bookmark level for unknown mproof defaults to 0 on input + line 184. + + +[10]) + +[11] +\openout2 = `"(4) - experiments/experiments.aux"'. + + (./(4) - experiments/experiments.tex +Chapter 4. + +Underfull \hbox (badness 10000) in paragraph at lines 14--15 +[]\OT1/cmr/m/n/12 All tests have been ex-e-cuted with the bench-mark li-brary + [] + + + +[12 + + + + +] +<(4) - experiments/images/QR-lambda-error.png, id=226, 939.51pt x 650.43pt> +File: (4) - experiments/images/QR-lambda-error.png Graphic file (type png) + +Package pdftex.def Info: (4) - experiments/images/QR-lambda-error.png used on +input line 27. +(pdftex.def) Requested size: 200.75996pt x 138.98619pt. +<(4) - experiments/images/QR-forward_error.png, id=227, 939.51pt x 650.43pt> +File: (4) - experiments/images/QR-forward_error.png Graphic file (type png) + +Package pdftex.def Info: (4) - experiments/images/QR-forward_error.png used on + input line 34. +(pdftex.def) Requested size: 200.75996pt x 138.98619pt. + + +[13 <./(4) - experiments/images/QR-lambda-error.png> <./(4) - experiments/image +s/QR-forward_error.png>] +<(4) - experiments/images/LBFGS-iterations-gradient-ill.png, id=239, 939.51pt x + 650.43pt> +File: (4) - experiments/images/LBFGS-iterations-gradient-ill.png Graphic file ( +type png) + +Package pdftex.def Info: (4) - experiments/images/LBFGS-iterations-gradient-ill +.png used on input line 68. +(pdftex.def) Requested size: 200.75996pt x 138.98619pt. +<(4) - experiments/images/LBFGS-iterations-gradient-well.png, id=240, 939.51pt +x 650.43pt> +File: (4) - experiments/images/LBFGS-iterations-gradient-well.png Graphic file +(type png) + +Package pdftex.def Info: (4) - experiments/images/LBFGS-iterations-gradient-wel +l.png used on input line 74. +(pdftex.def) Requested size: 200.75996pt x 138.98619pt. +<(4) - experiments/images/LBFGS-LS-gradient-comparison.png, id=241, 939.51pt x +650.43pt> +File: (4) - experiments/images/LBFGS-LS-gradient-comparison.png Graphic file (t +ype png) + +Package pdftex.def Info: (4) - experiments/images/LBFGS-LS-gradient-comparison. +png used on input line 85. +(pdftex.def) Requested size: 313.69025pt x 217.16344pt. + + +[14 <./(4) - experiments/images/LBFGS-iterations-gradient-ill.png> <./(4) - exp +eriments/images/LBFGS-iterations-gradient-well.png>] +<(4) - experiments/images/QRvsLBFGS-scalability-time-illcond-n.png, id=251, 794 +.97pt x 578.16pt> +File: (4) - experiments/images/QRvsLBFGS-scalability-time-illcond-n.png Graphic + file (type png) + +Package pdftex.def Info: (4) - experiments/images/QRvsLBFGS-scalability-time-il +lcond-n.png used on input line 100. +(pdftex.def) Requested size: 200.75996pt x 146.00409pt. +<(4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-n.png, id=252, 79 +4.97pt x 578.16pt> +File: (4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-n.png Graphi +c file (type png) + +Package pdftex.def Info: (4) - experiments/images/QRvsLBFGS-scalability-time-we +llcond-n.png used on input line 106. +(pdftex.def) Requested size: 200.75996pt x 146.00409pt. + + +[15 <./(4) - experiments/images/LBFGS-LS-gradient-comparison.png>] +<(4) - experiments/images/QRvsLBFGS-scalability-time-illcond-m.png, id=261, 794 +.97pt x 578.16pt> +File: (4) - experiments/images/QRvsLBFGS-scalability-time-illcond-m.png Graphic + file (type png) + +Package pdftex.def Info: (4) - experiments/images/QRvsLBFGS-scalability-time-il +lcond-m.png used on input line 119. +(pdftex.def) Requested size: 200.75996pt x 146.00409pt. +<(4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-m.png, id=262, 79 +4.97pt x 578.16pt> +File: (4) - experiments/images/QRvsLBFGS-scalability-time-wellcond-m.png Graphi +c file (type png) + +Package pdftex.def Info: (4) - experiments/images/QRvsLBFGS-scalability-time-we +llcond-m.png used on input line 125. +(pdftex.def) Requested size: 200.75996pt x 146.00409pt. + + +[16 <./(4) - experiments/images/QRvsLBFGS-scalability-time-illcond-n.png> <./(4 +) - experiments/images/QRvsLBFGS-scalability-time-wellcond-n.png> <./(4) - expe +riments/images/QRvsLBFGS-scalability-time-illcond-m.png> <./(4) - experiments/i +mages/QRvsLBFGS-scalability-time-wellcond-m.png>] +<(4) - experiments/images/LBFGS-iterations-memory-ill.png, id=273, 939.51pt x 6 +50.43pt> +File: (4) - experiments/images/LBFGS-iterations-memory-ill.png Graphic file (ty +pe png) + +Package pdftex.def Info: (4) - experiments/images/LBFGS-iterations-memory-ill.p +ng used on input line 144. +(pdftex.def) Requested size: 200.75996pt x 138.98619pt. +<(4) - experiments/images/LBFGS-iterations-memory-well.png, id=274, 939.51pt x +650.43pt> +File: (4) - experiments/images/LBFGS-iterations-memory-well.png Graphic file (t +ype png) + +Package pdftex.def Info: (4) - experiments/images/LBFGS-iterations-memory-well. +png used on input line 150. +(pdftex.def) Requested size: 200.75996pt x 138.98619pt. + + +[17 <./(4) - experiments/images/LBFGS-iterations-memory-ill.png> <./(4) - exper +iments/images/LBFGS-iterations-memory-well.png>] +<(4) - experiments/images/BFGS-LBFGS-gradient-comparison.png, id=290, 939.51pt +x 650.43pt> +File: (4) - experiments/images/BFGS-LBFGS-gradient-comparison.png Graphic file +(type png) + +Package pdftex.def Info: (4) - experiments/images/BFGS-LBFGS-gradient-compariso +n.png used on input line 170. +(pdftex.def) Requested size: 200.75996pt x 138.98619pt. +<(4) - experiments/images/BFGSvsLBFGS-time-m.png, id=291, 794.97pt x 578.16pt> +File: (4) - experiments/images/BFGSvsLBFGS-time-m.png Graphic file (type png) + +Package pdftex.def Info: (4) - experiments/images/BFGSvsLBFGS-time-m.png used +on input line 176. +(pdftex.def) Requested size: 192.39949pt x 139.92574pt. + + +[18] +<(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond-Dogleg.png, id= +302, 794.97pt x 578.16pt> +File: (4) - experiments/images/Quasi-Newton-Comparison-time-wellcond-Dogleg.png + Graphic file (type png) + + +Package pdftex.def Info: (4) - experiments/images/Quasi-Newton-Comparison-time- +wellcond-Dogleg.png used on input line 195. +(pdftex.def) Requested size: 200.75996pt x 146.00409pt. +<(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond.png, id=303, 79 +4.97pt x 578.16pt> +File: (4) - experiments/images/Quasi-Newton-Comparison-time-wellcond.png Graphi +c file (type png) + +Package pdftex.def Info: (4) - experiments/images/Quasi-Newton-Comparison-time- +wellcond.png used on input line 202. +(pdftex.def) Requested size: 200.75996pt x 146.00409pt. + + +[19 <./(4) - experiments/images/BFGS-LBFGS-gradient-comparison.png> <./(4) - ex +periments/images/BFGSvsLBFGS-time-m.png>] +<(4) - experiments/images/Quasi-Newton-Comparison-memory-wellcond.png, id=313, +794.97pt x 578.16pt> +File: (4) - experiments/images/Quasi-Newton-Comparison-memory-wellcond.png Grap +hic file (type png) + +Package pdftex.def Info: (4) - experiments/images/Quasi-Newton-Comparison-memor +y-wellcond.png used on input line 217. +(pdftex.def) Requested size: 271.86234pt x 197.71866pt. +) + +[20 <./(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond-Dogleg.pn +g> <./(4) - experiments/images/Quasi-Newton-Comparison-time-wellcond.png> <./(4 +) - experiments/images/Quasi-Newton-Comparison-memory-wellcond.png>] +\openout2 = `"(6) - conclusion/conclusion.aux"'. + + +(./(6) - conclusion/conclusion.tex +Chapter 5. +) + +[21 + + + + +] +\openout2 = `"(7) - proofs/proofs.aux"'. + + (./(7) - proofs/proofs.tex +Chapter 6. +Package hyperref Info: bookmark level for unknown mlemma defaults to 0 on input + line 3. +) + +[22 + + + + +] + +[23 + + + +] (./main.aux (./(1) - introduction/introduction.aux) +(./(2) - problem definition/problem definition.aux) +(./(3) - algorithms/algorithms.aux) (./(4) - experiments/experiments.aux) +(./(6) - conclusion/conclusion.aux) (./(7) - proofs/proofs.aux)) + *********** +LaTeX2e <2024-06-01> patch level 1 +L3 programming layer <2024-05-27> + *********** +Package logreq Info: Writing requests to 'main.run.xml'. +\openout1 = `main.run.xml'. + + ) +Here is how much of TeX's memory you used: + 50750 strings out of 473600 + 1154121 string characters out of 5728702 + 2110547 words of memory out of 5000000 + 72708 multiletter control sequences out of 15000+600000 + 586264 words of font info for 137 fonts, out of 8000000 for 9000 + 1141 hyphenation exceptions out of 8191 + 123i,20n,169p,1054b,1821s stack positions out of 10000i,1000n,20000p,200000b,200000s + +Output written on main.pdf (25 pages, 1575533 bytes). +PDF statistics: + 552 PDF objects out of 1000 (max. 8388607) + 413 compressed objects within 5 object streams + 132 named destinations out of 1000 (max. 500000) + 239 words of extra memory for PDF output out of 10000 (max. 10000000) + diff --git a/Report/main.pdf b/Report/main.pdf new file mode 100644 index 0000000..23976de Binary files /dev/null and b/Report/main.pdf differ diff --git a/Report/main.run.xml b/Report/main.run.xml new file mode 100644 index 0000000..2c685c2 --- /dev/null +++ b/Report/main.run.xml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + +]> + + + latex + + main.bcf + + + main.bbl + + + blx-dm.def + blx-compat.def + biblatex.def + standard.bbx + numeric.bbx + numeric-comp.bbx + ieee.bbx + numeric-comp.cbx + ieee.cbx + biblatex.cfg + english.lbx + + + + biber + + biber + main + + + main.bcf + + + main.bbl + + + main.bbl + + + main.bcf + + + document.bib + + + diff --git a/Report/main.synctex.gz b/Report/main.synctex.gz new file mode 100644 index 0000000..a4e7a2c Binary files /dev/null and b/Report/main.synctex.gz differ diff --git a/Report/main.tex b/Report/main.tex new file mode 100644 index 0000000..312b4f2 --- /dev/null +++ b/Report/main.tex @@ -0,0 +1,292 @@ +\documentclass[12pt]{report} +%% - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - %% +%% Load Packages %% +%% - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - %% +\usepackage[ + %top=2cm, + %bottom=2cm, + %left=2cm, + %right=2cm, + headheight=20pt, + centering +]{geometry} +\geometry{a4paper} + + +\usepackage[utf8]{inputenc} %% use UTF-8, maybe not needed since 2018 +\usepackage[english]{babel} %% language + +\pagestyle{headings} + +\usepackage{scrlayer-scrpage} + +\ifoot[]{} +\cfoot[]{} +\ofoot[\pagemark]{\pagemark} +\pagestyle{scrplain} + +\usepackage[ + backend=biber, + style=ieee, + sorting=ynt +]{biblatex} %% for citations +\addbibresource{document.bib} + +\usepackage{import} %% specify path for import + +%% math packages +\usepackage{graphicx} %% for pictures +\usepackage{float} +\usepackage{amssymb} %% math symbols +\usepackage{amsmath} %% math matrix etc +\usepackage{tabularray} %% better tables +\usepackage{booktabs} %% rules for tables +\usepackage{mathrsfs} +\usepackage{mathtools} +\usepackage{algpseudocode} %% loads algorithmicx +\usepackage{amsthm} +\usepackage{thmtools} %% theorems + +%% plot packages +\usepackage{pgfplots} %% plots used with \begin{tikzpicture} +\usepackage{tikz} %% for pictures +\usetikzlibrary{trees} +\usetikzlibrary{matrix, positioning, fit} +\pgfplotsset{width=10cm,compat=newest} + +%% design packages +\usepackage{enumitem} %% for lists and enumerating +\usepackage{color} +\usepackage{xcolor,colortbl} % xcolor for defining colors, colortbl for table colors +\usepackage{makecell} %% for multiple lines in cell of table +\usepackage{cancel} +\usepackage{pgfornament} %% ornaments +\usepackage{multicol} +\usepackage{subcaption} +\usepackage{bold-extra} + +%% load last +\usepackage{csquotes} %% correct language also for citations +\usepackage[hidelinks]{hyperref} %% links for table of contents, load last +\usepackage{bookmark} %% for better table of contents + + +%% - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - %% +%% Configuration of the packages %% +%% - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - %% + +\linespread{1} +\raggedbottom %% spaces if page is empty % chktex 1 + +%% set max table of contents recursion to subsection (3->subsubsecition) +\setcounter{tocdepth}{3} +\setcounter{secnumdepth}{3} + +%% use bar instead of arrow for vectors +\renewcommand{\vec}[1]{\bar{#1}} +%% easy norm +\newcommand{\norm}[1]{\left\lVert#1\right\rVert} + +\DeclareMathOperator{\Tr}{tr} + +% argmin and argmax +\DeclareMathOperator*{\argmax}{argmax} +\DeclareMathOperator*{\argmin}{argmin} +\DeclareMathOperator{\diag}{diag} + +%% itemize use less vertical space (use olditemize for default behaviour) +\let\olditemize=\itemize%% old itemize +\let\endolditemize=\enditemize%% old end itemize +\renewenvironment{itemize}{\olditemize\itemsep-0.2em}{\endolditemize} + +%% items in itemize emph+box +%% usage: \ieb{Class:} for simple item +%% \ieb[4cm]{Class:} for specific size of box +\newcommand{\ieb}[2][2cm]{ + \makebox[#1][l]{\emph{#2}} +} %% TODO: replace with description environment (? maybe) + +% less vertical space around align & align* +\newcommand{\zerodisplayskips}{ + \setlength{\abovedisplayskip}{0pt} + \setlength{\belowdisplayskip}{0pt} + \setlength{\abovedisplayshortskip}{0pt} + \setlength{\belowdisplayshortskip}{0pt} +} + +% make dotfill use all the space available +\renewcommand{\dotfill}{ + \leavevmode\cleaders\hbox to 1.00em{\hss .\hss }\hfill\kern0pt } % chktex 1 chktex 26 + +\setlength{\fboxsep}{-\fboxrule} % for debugging + + +%% PACKAGE algorithm +\usepackage[ruled,vlined,linesnumbered]{algorithm2e} + + +%% PACKAGE tabularray +\UseTblrLibrary{amsmath} + + +%% PACKAGE color +\definecolor{red}{rgb}{1, 0.1, 0.1} +\definecolor{lightgreen}{rgb}{0.55, 0.87, 0.47} +\definecolor{gray}{rgb}{0.3, 0.3, 0.3} +\newcommand{\lgt}{\cellcolor{lightgreen}} %% light green in tables +\newcommand{\gry}{\textcolor{gray}} %% gray text +\newcommand{\rd}{\textcolor{red}} %% red text + +%% PACKAGE minipage +\newcommand{\thend}[1]{\begin{center} + \begin{minipage}[c][1em][c]{#1} + \dotfill{} + \end{minipage} +\end{center}} + + +%% PACKAGE thmtools +\declaretheoremstyle[ + headfont=\normalfont\bfseries, + notefont=\mdseries, + bodyfont=\normalfont, + qed=\qedsymbol{} +]{steo} +\declaretheorem[name={Theorem}, numbered=no, style=steo]{mtheo} + +\declaretheoremstyle[ + headfont=\normalfont\bfseries, + notefont=\mdseries, + bodyfont=\normalfont, + qed=\qedsymbol{} +]{slemma} +\declaretheorem[name={Lemma}, numbered=yes, style=slemma]{mlemma} + +\declaretheoremstyle[ + headfont=\normalfont\bfseries, + notefont=\mdseries, + bodyfont=\normalfont, +]{sdef} +\declaretheorem[numbered=no, style=sdef]{mdef} + +\declaretheoremstyle[ + spaceabove=-6pt, + spacebelow=6pt, + headfont=\normalfont\bfseries, + bodyfont=\normalfont, + postheadspace=1em, + qed=$\blacksquare$, + headpunct={:} +]{sprf} +\declaretheorem[name={Proof}, style=sprf, numbered=no]{mproof} + +% -------------------- Fancy style -------------------- + +% \usepackage{fancyhdr} +% \pagestyle{fancy} +% \renewcommand{\chaptermark}[1]{\markboth{#1}{}} +% \setlength{\headheight}{22pt} +% \fancyhead[R]{\leftmark} +% \fancyhead[L]{\smallLogoA} +% \setlength{\parskip}{0.5em} + + +\usepackage{titlesec} +\titlespacing*{\chapter}{0pt}{-20pt}{20pt} +\titleformat{\chapter}[display]{\normalfont\bfseries}{}{0pt}{\Huge} + + +%% ......................................................................... %% +%% local changes +% \setcounter{secnumdepth}{0} + + +%% - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - %% + +\title{Document} +\date{\today} + +%% - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - %% +\newcommand{\logoA}{\includegraphics[height=4cm]{images/UniPi Logo.png}} +\newcommand{\smallLogoA}{\includegraphics[height=0.6cm]{images/UniPi Logo.png}} +\newcommand{\course}{Computational Mathematics for Learning and Data Analysis} +\newcommand{\assignment}{Project Track 19 --- Non-ML} +\newcommand{\authors}{ + \begin{tblr}{colspec={c}, cells={cmd={\bf\textit}}} + De Castelli Fabrizio \\ + Rossi Elvis + \end{tblr} +} + +\newcommand{\emails}{ +\begin{tblr}{colspec = {rr}, colsep={0pt}, cells={cmd={\textit}}, rowsep={4pt}} + f.decastelli&\gry{@studenti.unipi.it} \\ + e.rossi46&\gry{@studenti.unipi.it} +\end{tblr} +} +\newcommand{\universityA}{Università di Pisa} + +\newcommand{\academicYear}{Academic Year 2023--2024} +\newcommand{\department}{Department of Computer Science} + +% Inter-word Spacing +\usepackage{setspace} +\setstretch{1.25} + +\begin{document} + +\begin{titlepage} + +\centering +\logoA\par +\vspace{0.7cm} +{\scshape\LARGE \universityA\par} + +\vspace{2cm} +{\scshape\Huge \textbf{\course} \par} + +\vspace{1cm} +{\scshape\LARGE \assignment\par} + +\vfill + +{\scshape\LARGE \textbf{Team 16} \par} +\vspace{0.5cm} +\begin{tblr}{colspec = {Q[m,c]Q[m,c]}, colsep={6pt}} + {\Large \authors} & {\large \emails} +\end{tblr} \\ +\vspace{1cm} +{\large \academicYear\par} + +\end{titlepage} + +\pagenumbering{roman} +\setcounter{page}{1} + +% Table of contents +\tableofcontents + +\clearpage +\pagenumbering{arabic} +\setcounter{page}{1} + + +\begin{sloppypar} + % Chapters + \include{(1) - introduction/introduction} + \include{(2) - problem definition/problem definition} + \include{(3) - algorithms/algorithms} + \include{(4) - experiments/experiments} + \include{(6) - conclusion/conclusion} + \include{(7) - proofs/proofs} +\end{sloppypar} + +\printbibliography{} + +\end{document} + +%% - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - %% + +%%% Local Variables: +%%% TeX-command-extra-options: "-shell-escape" +%%% End: diff --git a/Report/main.toc b/Report/main.toc new file mode 100644 index 0000000..2e75b4b --- /dev/null +++ b/Report/main.toc @@ -0,0 +1,18 @@ +\babel@toc {english}{}\relax +\contentsline {chapter}{\numberline {1}Introduction}{1}{chapter.1}% +\contentsline {chapter}{\numberline {2}Problem Definition}{2}{chapter.2}% +\contentsline {section}{\numberline {2.1}QR}{2}{section.2.1}% +\contentsline {section}{\numberline {2.2}L-BFGS}{3}{section.2.2}% +\contentsline {section}{\numberline {2.3}Conditioning}{4}{section.2.3}% +\contentsline {chapter}{\numberline {3}Algorithms}{6}{chapter.3}% +\contentsline {section}{\numberline {3.1}QR}{6}{section.3.1}% +\contentsline {section}{\numberline {3.2}L-BFGS}{8}{section.3.2}% +\contentsline {chapter}{\numberline {4}Experiments}{12}{chapter.4}% +\contentsline {section}{\numberline {4.1}QR}{13}{section.4.1}% +\contentsline {section}{\numberline {4.2}L-BFGS}{14}{section.4.2}% +\contentsline {section}{\numberline {4.3}Comparison between QR and L-BFGS}{15}{section.4.3}% +\contentsline {section}{\numberline {4.4}Other Experiments}{17}{section.4.4}% +\contentsline {subsection}{\numberline {4.4.1}The Effect of the Memory Size}{17}{subsection.4.4.1}% +\contentsline {subsection}{\numberline {4.4.2}A Comparison of Quasi-Newton Methods}{18}{subsection.4.4.2}% +\contentsline {chapter}{\numberline {5}Concluding Remarks}{21}{chapter.5}% +\contentsline {chapter}{\numberline {6}Proofs}{22}{chapter.6}% diff --git a/project/L-BFGS/BFGS.jl b/project/L-BFGS/BFGS.jl new file mode 100644 index 0000000..7babb26 --- /dev/null +++ b/project/L-BFGS/BFGS.jl @@ -0,0 +1,314 @@ +module BFGS + +using LinearAlgebra: norm, I, dot, diagm, mul! + +using ..OracleFunction + +export BroydenFletcherGoldfarbShanno, BroydenFletcherGoldfarbShannoDogleg + +const armijiowolfeorexact = :exact +BFGSorDFP = :BFGS + +function ArmijoWolfeLineSearch( + f::Union{LeastSquaresF, OracleF}, + x::AbstractArray, + p::AbstractArray, + MaxEvaluations::Integer; + αinit::Real=1, + τ::Real=1.1, + c1::Real=1e-4, + c2::Real=0.9, + ϵα::Real=1e-16, + ϵgrad::Real=1e-12, + safeguard::Real=0.20, + )::Tuple{Real, Integer} + + ϕ = (α) -> begin + v = f.eval(x + α * p) + gradient = f.grad(x + α * p) + return (v, dot(p, gradient)) + end + + α = αinit + local αgrad + + ϕ_0, ϕd_0 = ϕ(0) + + while MaxEvaluations > 0 + αcurr, αgrad = ϕ(α) + MaxEvaluations -= 2 + + if (αcurr ≤ ϕ_0 + c1 * α * ϕd_0) && (abs(αgrad) ≤ -c2 * ϕd_0) + return (α, MaxEvaluations) + end + + if αgrad ≥ 0 + break + end + α *= τ + end + + αlo = 0 + αhi = α + αlograd = ϕd_0 + αhigrad = αgrad + + while (MaxEvaluations > 0) && (αhi - αlo) > ϵα && (αgrad > ϵgrad) + α = (αlo * αhigrad - αhi * αlograd)/(αhigrad - αlograd) + α = max( + αlo + (αhi - αlo) * safeguard, + min(αhi - (αhi - αlo) * safeguard, α) + ) + + αcurr, αgrad = ϕ(α) + MaxEvaluations -= 2 + + if (αcurr ≤ ϕ_0 + c1 * α * ϕd_0) && (abs(αgrad) ≤ -c2 * ϕd_0) + break + end + + if αgrad < 0 + αlo = α + αlograd = αgrad + else + αhi = α + if αhi ≤ ϵα + break + end + αhigrad = αgrad + end + end + + return (α, MaxEvaluations) +end + +function ExactLineSearch( + f::LeastSquaresF, + x::AbstractArray, + p::AbstractArray, + MaxEvaluations::Integer + ) + MaxEvaluations -= 1 + return (tomography(f, x, p), MaxEvaluations) +end + + +@doc raw""" +```julia +BroydenFletcherGoldfarbShanno(f::Union{LeastSquaresF, OracleF}, [x::AbstractVector{T}, ϵ::T=1e-6, MaxEvaluations::Integer=10000]) +``` + +Computes the minimum of the input function `f`. + +### Input + +- `f` -- the input function to minimize. +- `x` -- the starting point, if not specified the default one for the function `f` is used. +- `ϵ` -- the tollerance for the stopping criteria. +- `m` -- maximum number of vector to store that compute the approximate hessian. +- `MaxEvaluations` -- maximum number of function evaluations. Both ```f.eval``` and ```f.grad``` are counted. + +### Output + +A named tuple containing: +- `x` -- the minimum found +- `eval` -- the value of the function at the minimum +- `grad` -- the gradient of the function at the minimum +- `RemainingEvaluations` -- the number of function evaluation not used. + +""" +function BroydenFletcherGoldfarbShanno( + f::Union{LeastSquaresF, OracleF}; + x::Union{Nothing, AbstractVector{T}}=nothing, + ϵ::T=1e-6, + MaxEvaluations::Integer=10000 + )::NamedTuple where {T} + + if isnothing(x) + x = f.starting_point + end + + gradient = f.grad(x) + MaxEvaluations -= 1 + normgradient0 = norm(gradient) + H = diagm(ones(length(x))) + tmp1 = similar(H) + tmp2 = similar(H) + + firstEvaluation = true + + while MaxEvaluations > 0 && norm(gradient) > ϵ * normgradient0 + p = -H * gradient # direction + + α, MaxEvaluations = + if armijiowolfeorexact === :armijiowolfe || f isa OracleF + ArmijoWolfeLineSearch(f, x, p, MaxEvaluations) + elseif armijiowolfeorexact === :exact + ExactLineSearch(f, x, p, MaxEvaluations) + end + + previousx = x + x = x + α * p + + previousgradient = gradient + gradient = f.grad(x) + MaxEvaluations -= 1 + + s = x - previousx + y = gradient - previousgradient + ρ = inv(dot(y, s)) + + # if its the first iteration then set H to an aproximation of the Hessian + if firstEvaluation + mul!(H, I, dot(y, s)/dot(y, y)) + firstEvaluation = false + end + + if BFGSorDFP == :DFP + # DFP update ------------------------------------------- + # H = H - (H * y * y' * H)/(y' * H * y) + (s * s')/(y' * s) + + mul!(tmp1, H * y * y', H) + mul!(tmp2, s, s') + H .+= -tmp1/dot(y, H, y) .+ ρ * tmp2 + elseif BFGSorDFP == :BFGS + # BFGS update ------------------------------------------ + # H = (I - ρ * s * y') * H * (I - ρ * y * s') + ρ * s * s' + + mul!(tmp1, H * y, s') + mul!(tmp2, s, s') + H .+= ρ * ((1 + ρ * dot(y, H, y)) .* tmp2 .- tmp1 .- tmp1') + end + end + + return (; + :x => x, + :eval => f.eval(x), + :grad => gradient, + :RemainingEvaluations => MaxEvaluations) +end + + +function BroydenFletcherGoldfarbShannoDogleg( + f::Union{LeastSquaresF, OracleF}; + x::Union{Nothing, AbstractVector{T}}=nothing, + ϵ::T=1e-6, + MaxEvaluations::Integer=10000 + )::NamedTuple where {T} + + if isnothing(x) + x = f.starting_point + end + + Δ = 1 # initial size of trust region + smallestΔ = 1e-4 # smallest size where linear aproximation is applied + + gradient = f.grad(x) + MaxEvaluations -= 1 + normgradient0 = norm(gradient) + normgradient = normgradient0 + H = diagm(ones(length(x))) + B = copy(H) + tmp1 = similar(H) + tmp2 = similar(H) + tmp3 = similar(H) + + firstEvaluation = true + + while MaxEvaluations > 0 && norm(gradient) > ϵ * normgradient0 + # compute s by solving the subproblem min_s grad' * s + 0.5 s' * B * s with norm(s) ≤ Δ + CauchyPoint = -(Δ/normgradient) * gradient + τ = if dot(gradient, B, gradient) ≤ 0 + 1 + else + min((normgradient^3)/(Δ * dot(gradient, B, gradient)), 1) + end + + if Δ ≤ smallestΔ || B == I + # the Cauchy point is enought for small regions (linear aproximation) + s = τ * CauchyPoint + else + pB = -H * gradient + pU = -dot(gradient, gradient)/dot(gradient, B, gradient) * gradient + + if norm(pB) ≤ Δ + # the region is larger than the dogleg + s = pB + elseif Δ ≤ norm(pU) + # the region is smaller than the first step + s = Δ/norm(pU) * pU + else + # solve the quadratic sistem for the dogleg + one = dot(pU, (pB - pU)) + two = dot(pB - pU, pB - pU) + three = dot(pU, pU) + + τ = (-one+two + sqrt(one^2 - three * two + two * Δ^2))/two + s = pU + (τ - 1) * (pB - pU) + end + end + + previousx = x + x = x + s + + previousgradient = gradient + gradient = f.grad(x) + normgradient = norm(gradient) + MaxEvaluations -= 1 + + y = gradient - previousgradient + ρ = inv(dot(y, s)) + + ared = f.eval(x) - f.eval(x + s) # actual reduction + pred = -(dot(gradient, s) + 0.5 * dot(s, B, s)) # predicted reduction + MaxEvaluations -= 2 + + # expand or contract the region + if (0.75 < ared/pred) && (0.8 * Δ < norm(s)) + Δ = 2 * Δ + elseif (ared/pred < 0.1) + Δ = 0.5 * Δ + end + + # if its the first iteration then set H to an aproximation of the Hessian + if firstEvaluation + mul!(H, I, dot(y, s)/dot(y, y)) + firstEvaluation = false + end + + if BFGSorDFP == :DFP + # DFP update ------------------------------------------- + # H = H - (H * y * y' * H)/(y' * H * y) + (s * s')/(y' * s) + + mul!(tmp1, H * y * y', H) + mul!(tmp2, s, s') + H .+= -tmp1/dot(y, H, y) .+ ρ * tmp2 + + mul!(tmp1, y, s') + tmp2 = I - ρ * tmp1 + mul!(tmp1, tmp2, B) + mul!(tmp3, tmp1, tmp2') + mul!(tmp2, y, y') + B .= tmp3 .+ ρ * tmp2 + elseif BFGSorDFP == :BFGS + # BFGS update ------------------------------------------ + # H = (I - ρ * s * y') * H * (I - ρ * y * s') + ρ * s * s' + + mul!(tmp1, H * y, s') + mul!(tmp2, s, s') + H .+= ρ * ((1 + ρ * dot(y, H, y)) .* tmp2 .- tmp1 .- tmp1') + + mul!(tmp1, B * s * s', B) + mul!(tmp2, y, y') + B .+= -tmp1/dot(s, B, s) .+ ρ * tmp2 + end + end + + return (; + :x => x, + :eval => f.eval(x), + :grad => gradient, + :RemainingEvaluations => MaxEvaluations) +end + +end # module BFGS \ No newline at end of file diff --git a/project/L-BFGS/LBFGS.jl b/project/L-BFGS/LBFGS.jl new file mode 100644 index 0000000..5b52a25 --- /dev/null +++ b/project/L-BFGS/LBFGS.jl @@ -0,0 +1,193 @@ +module LBFGS + +using LinearAlgebra: norm, I, dot +using DataStructures: CircularBuffer + +using ..OracleFunction + +export LimitedMemoryBFGS + +const armijiowolfeorexact = :exact + +function ArmijoWolfeLineSearch( + f::Union{LeastSquaresF, OracleF}, + x::AbstractArray, + p::AbstractArray, + MaxEvaluations::Integer; + αinit::Real=1, + τ::Real=1.1, + c1::Real=1e-4, + c2::Real=0.9, + ϵα::Real=1e-16, + ϵgrad::Real=1e-12, + safeguard::Real=0.20, + )::Tuple{Real, Integer} + + ϕ = (α) -> begin + v = f.eval(x + α * p) + gradient = f.grad(x + α * p) + return (v, dot(p, gradient)) + end + + α = αinit + local αgrad + + ϕ_0, ϕd_0 = ϕ(0) + + while MaxEvaluations > 0 + αcurr, αgrad = ϕ(α) + MaxEvaluations -= 2 + + if (αcurr ≤ ϕ_0 + c1 * α * ϕd_0) && (abs(αgrad) ≤ -c2 * ϕd_0) + return (α, MaxEvaluations) + end + + if αgrad ≥ 0 + break + end + α *= τ + end + + αlo = 0 + αhi = α + αlograd = ϕd_0 + αhigrad = αgrad + + while (MaxEvaluations > 0) && (αhi - αlo) > ϵα && (αgrad > ϵgrad) + α = (αlo * αhigrad - αhi * αlograd)/(αhigrad - αlograd) + α = max( + αlo + (αhi - αlo) * safeguard, + min(αhi - (αhi - αlo) * safeguard, α) + ) + + αcurr, αgrad = ϕ(α) + MaxEvaluations -= 2 + + if (αcurr ≤ ϕ_0 + c1 * α * ϕd_0) && (abs(αgrad) ≤ -c2 * ϕd_0) + break + end + + if αgrad < 0 + αlo = α + αlograd = αgrad + else + αhi = α + if αhi ≤ ϵα + break + end + αhigrad = αgrad + end + end + + return (α, MaxEvaluations) +end + +function ExactLineSearch( + f::LeastSquaresF, + x::AbstractArray, + p::AbstractArray, + MaxEvaluations::Integer + ) + MaxEvaluations -= 1 + return (tomography(f, x, p), MaxEvaluations) +end + +@doc raw""" +```julia +LimitedMemoryBFGS(f::Union{LeastSquaresF{T}, OracleF{T, F, G}}, [x::AbstractVector{T}, ϵ::T=1e-6, m::Integer=3, MaxEvaluations::Integer=10000]) +``` + +Computes the minimum of the input function `f`. + +### Input + +- `f` -- the input function to minimize. +- `x` -- the starting point, if not specified the default one for the function `f` is used. +- `ϵ` -- the tollerance for the stopping criteria. +- `m` -- maximum number of vector to store that compute the approximate hessian. +- `MaxEvaluations` -- maximum number of function evaluations. Both ```f.eval``` and ```f.grad``` are counted. + +### Output + +A named tuple containing: +- `x` -- the minimum found +- `eval` -- the value of the function at the minimum +- `grad` -- the gradient of the function at the minimum +- `RemainingEvaluations` -- the number of function evaluation not used. + +See also [`QRhous`](@ref). +""" +function LimitedMemoryBFGS( + f::Union{LeastSquaresF, OracleF}; + x::Union{Nothing, AbstractVector{T}}=nothing, + ϵ::T=1e-6, + m::Integer=3, + MaxEvaluations::Integer=10000 + )::NamedTuple where {T} + + if isnothing(x) + x = f.starting_point + end + + gradient = f.grad(x) + MaxEvaluations -= 1 + normgradient0 = norm(gradient) + H = CircularBuffer{NamedTuple}(m) + αstore = Array{eltype(x)}(undef, 0) + + while MaxEvaluations > 0 && norm(gradient) > ϵ * normgradient0 + # two loop recursion for finding the direction + q = gradient + empty!(αstore) + + for i ∈ reverse(H) + push!(αstore, i[:ρ] * dot(i[:s], q)) + q -= αstore[end] * i[:y] + end + # choose H0 as something resembling the hessian + H0 = if isempty(H) + I + else + (dot(H[end][:s], H[end][:y])/dot(H[end][:y], H[end][:y])) * I + end + r = H0 * q + for i ∈ H + βi = i[:ρ] * dot(i[:y], r) + r += i[:s] * (pop!(αstore) - βi) + end + p = -r # direction + + if armijiowolfeorexact === :armijiowolfe || f isa OracleF + α, MaxEvaluations = ArmijoWolfeLineSearch(f, x, p, MaxEvaluations) + elseif armijiowolfeorexact === :exact + α, MaxEvaluations = ExactLineSearch(f, x, p, MaxEvaluations) + end + + previousx = x + x = x + α * p + + previousgradient = gradient + gradient = f.grad(x) + MaxEvaluations -= 1 + + s = x - previousx + y = gradient - previousgradient + + curvature = dot(s, y) + ρ = inv(curvature) + + if curvature ≤ 1e-16 + empty!(H) # restart from the gradient + else + push!(H, (; :ρ => ρ, :y => y, :s => s)) + end + end + + return (; + :x => x, + :eval => f.eval(x), + :grad => gradient, + :RemainingEvaluations => MaxEvaluations) +end + +end # module LBGGS diff --git a/project/L-BFGS/OracleFunction.jl b/project/L-BFGS/OracleFunction.jl new file mode 100644 index 0000000..f1c72c4 --- /dev/null +++ b/project/L-BFGS/OracleFunction.jl @@ -0,0 +1,85 @@ +module OracleFunction + +using LinearAlgebra: norm + +export OracleF, LeastSquaresF, tomography + +@doc """ +```julia +OracleF{T, F<:Function, G<:Function} +``` + +Struct that holds a generic function to evaluate. +`eval` is the function that evaluates a point, `grad` is the gradient of the function and +`starting_point` is the point from which minimization should start. +""" +struct OracleF{T, F<:Function, G<:Function} + starting_point::AbstractArray{T} + eval::F + grad::G +end + +@doc """ +```julia +LeastSquaresF{T, F<:Function, G<:Function} +``` + +Struct that holds an instance of a least squares problem. The interface is similar to the `OracleF` struct. +`eval` is the function that evaluates a point, `grad` is the gradient of the function and +`starting_point` is the point from which minimization should start. + +See also [`OracleF`](@ref). +""" +struct LeastSquaresF{T, F<:Function, G<:Function} + oracle::OracleF{T, F, G} + X::AbstractMatrix{T} + y::AbstractArray{T} + symm::AbstractMatrix{T} + yX::AbstractArray{T} +end + +function LeastSquaresF(starting_point::AbstractArray{T}, X::AbstractMatrix{T}, y::AbstractArray{T}) where T + f(x) = norm(X * x - y)^2 + df(x) = 2 * X' * (X * x - y) + symm = X' * X + yX = y' * X + + o = OracleF(starting_point, f, df) + LeastSquaresF(o, X, y, symm, yX) +end + +function LeastSquaresF(t::NamedTuple) + if [:X_hat, :y_hat, :start] ⊈ keys(t) + throw(ArgumentError("Input tuple does not contain necessary values, found: " * string(keys(t)))) + end + starting_point, X, y = t[:start], t[:X_hat], t[:y_hat] + + LeastSquaresF(starting_point, X, y) +end + +@doc """ +```julia +tomography(l::LeastSquaresF{T, F, G}, w::AbstractArray{T}, p::AbstractArray{T}) +``` + +Function that returns the minimum of the function `l` along the plane in `w` and with direction `p`. + +See also [`LeastSquaresF`](@ref). +""" +function tomography(l::LeastSquaresF{T, F, G}, w::AbstractArray{T}, p::AbstractArray{T}) where {T, F, G} + (l.yX * p - w' * l.symm * p) * inv(p' * l.symm * p) +end + +function Base.getproperty(l::LeastSquaresF{T, F, G}, name::Symbol) where {T, F, G} + if name === :eval + return l.oracle.eval + elseif name === :grad + return l.oracle.grad + elseif name === :starting_point + return l.oracle.starting_point + else + getfield(l, name) + end +end + +end ## module OracleFunction diff --git a/project/L-BFGS/SR1.jl b/project/L-BFGS/SR1.jl new file mode 100644 index 0000000..75f8553 --- /dev/null +++ b/project/L-BFGS/SR1.jl @@ -0,0 +1,119 @@ +module SR1 + +using LinearAlgebra: norm, I, dot, diagm, mul! + +using ..OracleFunction + +export SymmetricRank1 + + +function SymmetricRank1( + f::Union{LeastSquaresF, OracleF}; + x::Union{Nothing, AbstractVector{T}}=nothing, + ϵ::T=1e-6, + η::T=1e-4, # threshold for ignoring direction + r::T=1e-8, # skipping rule for updating B and H + MaxEvaluations::Integer=10000 + )::NamedTuple where {T} + + Δ = 1 # initial size of trust region + smallestΔ = 1e-4 # smallest size where linear aproximation is applied + + if isnothing(x) + x = f.starting_point + end + + gradient = f.grad(x) + evalx = f.eval(x) + nextevalx = 0 + MaxEvaluations -= 2 + normgradient0 = norm(gradient) + normgradient = normgradient0 + B = diagm(ones(length(x))) + H = diagm(ones(length(x))) + tmp1 = similar(x) + tmp2 = similar(H) + + local s + + while MaxEvaluations > 0 && normgradient > ϵ * normgradient0 + # compute s by solving the subproblem min_s grad' * s + 0.5 s' * B * s with norm(s) ≤ Δ + CauchyPoint = - (Δ/normgradient) * gradient + τ = if gradient' * B * gradient ≤ 0 + 1 + else + min((normgradient^3)/(Δ * dot(gradient, B, gradient)), 1) + end + + if Δ ≤ smallestΔ || B == I + # the Cauchy point is enought for small regions (linear aproximation) + s = τ * CauchyPoint + else + pB = -H * gradient + pU = -dot(gradient, gradient)/dot(gradient, B, gradient) * gradient + + if norm(pB) ≤ Δ + # the region is larger than the dogleg + s = pB + elseif Δ ≤ norm(pU) + # the region is smaller than the first step + s = Δ/norm(pU) * pU + else + # solve the quadratic sistem for the dogleg + one = dot(pU, (pB - pU)) + two = dot((pB - pU), (pB - pU)) + three = dot(pU, pU) + + τ = (-one+two + sqrt(one^2 - three * two + two * Δ^2))/two + s = pU + (τ - 1) * (pB - pU) + end + end + + # ------ + y = f.grad(x + s) - gradient + + nextevalx = f.eval(x + s) + ared = evalx - nextevalx # actual reduction + pred = -(dot(gradient, s) + 0.5 * dot(s, B, s)) # predicted reduction + + MaxEvaluations -= 2 + + if ared/pred > η + x = x + s + evalx = nextevalx + gradient = f.grad(x) + normgradient = norm(gradient) + MaxEvaluations -= 1 + end + + # expand or contract the region + if (0.75 < ared/pred) && (0.8 * Δ < norm(s)) + Δ = 2 * Δ + elseif (ared/pred < 0.1) + Δ = 0.5 * Δ + end + if abs(s' * (y - B * s)) ≥ r * norm(s) * norm(y - B * s) # if the denominator is not too small + # B = B + ((y - B * s)*(y - B * s)')/((y - B * s)' * s) + mul!(tmp1, B, -s) + tmp1 .+= y + mul!(tmp2, tmp1, tmp1') + tmp2 ./= dot(tmp1, s) + B .+= tmp2 + + # H = H + ((s - H * y) * (s - H * y)')/((s - H * y)' * y) + mul!(tmp1, H, -y) + tmp1 += s + mul!(tmp2, tmp1, tmp1') + tmp2 ./= dot(tmp1, y) + H .+= tmp2 + end + end + + return (; + :x => x, + :eval => f.eval(x), + :grad => gradient, + :RemainingEvaluations => MaxEvaluations) +end + +end # module SR1 \ No newline at end of file diff --git a/project/Project.toml b/project/Project.toml new file mode 100644 index 0000000..9f3c075 --- /dev/null +++ b/project/Project.toml @@ -0,0 +1,16 @@ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +ColorSchemes = "35d6a980-a343-548e-a6ea-1d62b119f2f4" +Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +GenericLinearAlgebra = "14197337-ba66-59df-a3e3-ca00e7dcff7a" +IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a" +Interact = "c601a237-2ae4-5e1e-952c-7a85b0c7eef1" +IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153" +LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +ProfileSVG = "132c30aa-f267-4189-9183-c8a63c7e05e6" +TickTock = "9ff05d80-102d-5586-aa04-3a8bd1a90d20" diff --git a/project/QR/housQR.jl b/project/QR/housQR.jl new file mode 100644 index 0000000..f703e41 --- /dev/null +++ b/project/QR/housQR.jl @@ -0,0 +1,348 @@ +module housQR + +import Base: size, show, getproperty, getfield, propertynames, \, * +using LinearAlgebra: norm, I, triu, diagm, dot, mul! + +export QRhous, qrfact + +@doc raw""" +```julia +QRhous{T <: Real} +``` + +Struct that holds a QR factorization. The R factor is stored in +the upper triangular part of the matrix A. The diagonal of R is stored in the +vector d. The Q factor is computed lazily. +""" +mutable struct QRhous{T <: Real} + raw""" + `A` holds the R factor in the upper triangular part minus the diagonal + and the housholder vectors in the lower triangular. + """ + QR::AbstractVecOrMat{T} # R after the QR factorization + + raw""" + `d` holds the diagonal of the R factor. + """ + d::AbstractArray{T} # diagonal of R + + raw""" + `AQ` holds the cached value for Q, is `nothing` if not yet computed + """ + AQ::Union{Nothing, AbstractVecOrMat{T}} # Q + + raw""" + `AR` holds the cached value for R, is `nothing` if not yet computed + """ + AR::Union{Nothing, AbstractVecOrMat{T}} # R0 + + QRhous(QR, d, AQ=nothing, AR=nothing) = new{eltype(QR)}(QR, d, AQ, AR) +end + + +@doc raw""" +```julia +householder_vector(x::AbstractVecOrMat{T})::Tuple{AbstractVecOrMat{T}, T} +``` + +Computes a normalized vector u such that ``Hx = se_1, H = I - 2uu^T``. + +### Input + +- `x` -- the input vector + +### Output + +A tuple [u, s], where u is the householder vector of x and ``s = \big | x \big |`` + +""" +function householder_vector(x::AbstractVecOrMat{T})::Tuple{AbstractVecOrMat{T}, T} where T + s = norm(x) + if x[1] ≥ 0 + s = -s + end + u = copy(x) + u[1] -= s + u ./= norm(u) + return u, s +end + +@doc raw""" +```julia +qrfact(A::Matrix{T})::QRhous{T} +``` + +Computes the QR factorization of A: A = QR. + +### Input + +- 'A' -- the input matrix; ``A \in \mathbb{R}^{m \times n}``. + +### Output + +A `QRhous` object containing the QR factorization of A. + +See also [`QRhous`](@ref). +""" +function qrfact(A::Matrix{T})::QRhous{T} where T + (m, n) = size(A) + R = deepcopy(A) + d = zeros(min(n, m)) + + tmp = similar(R) + + for k ∈ 1:min(n, m) + @views (u, s) = householder_vector(R[k:m, k]) + # construct R + d[k] = s + @views mul!(tmp[k:m, k+1:n], 2*u, u' * R[k:m, k+1:n]) + @views R[k:m, k+1:n] .-= tmp[k:m, k+1:n] + R[k:m, k] .= u + end + return QRhous(R, d) +end + +@doc raw""" +```julia +qyhoust(A::QRhous{T}, y::AbstractArray{T}) +``` + +Computes the product ``Q_0^Ty,`` where ``Q_0`` is the ``n \times n`` (upper) part of the matrix Q of the QR factorization of A. + +### Input + +- `A` -- the QR factorization of A. +- `y` -- the vector to be multiplied by ``Q_0^T``. + +### Output + +The product ``Q_0^Ty``. + +See also [`qyhous`](@ref). +""" +function qyhoust(A::QRhous{T}, y::AbstractArray{T})::AbstractArray{T} where T + m, n = size(A.QR) + z = deepcopy(y) + tmp = similar(z) + + for j ∈ 1:n + @views tmp[j:m] .= 2 * dot(A.QR[j:m, j], z[j:m]) * A.QR[j:m, j] + @views z[j:m] .-= tmp[j:m] + end + return z +end + +@doc raw""" +```julia +qyhous(A::QRhous{T}, y::AbstractArray{T}) +``` + +Computes the product ``Q_0y,`` where ``Q_0`` is the ``n \times n`` (upper) part of the matrix Q of the QR factorization of A. + +### Input + +- `A` -- the QR factorization of A. +- `y` -- the vector to be multiplied by ``Q_0``. + +### Output + +The product ``Q_0y``. + +See also [`qyhoust`](@ref). +""" +function qyhous(A::QRhous{T}, y::AbstractArray{T}) where T + m, n = size(A.QR) + z = deepcopy(y) + tmp = similar(z) + + for j ∈ n:-1:1 + @views tmp[j:m] .= 2 * dot(A.QR[j:m, j], z[j:m]) * A.QR[j:m, j] + @views z[j:m] .-= tmp[j:m] + end + return z +end + +@doc raw""" +```julia +multiplybyr(A::QRhous{T}, y::AbstractArray{T}) +``` + +Computes the product ``Ry,`` where ``R`` is the upper part of the matrix A, already transformed by the QR factorization. + +""" +function multiplybyr(A::QRhous{T}, y::AbstractArray{T}) where T + m, n = size(A.QR) + min_size = min(m, n) + + vcat(A.R * y[1:min_size], zeros(max(0, m - n))) +end + +@doc raw""" +```julia +calculateQ(A::QRhous{T}) +``` + +Computes the Q factor of the QR factorization of A. The result is cached. + +### Input + +- `A` -- the QR factorization of `A`. + +### Output + +The Q factor of the QR factorization of `A`. + +""" +function calculateQ(A::QRhous{T}) where T + if !isnothing(A.AQ) + return A.AQ + end + m, n = size(A.QR) + + A.AQ = zeros(m, 0) + id = Matrix{eltype(A.QR)}(I, m, n) + for i ∈ eachcol(id) + A.AQ = [A.AQ qyhous(A, i)] + end + return A.AQ +end + +@doc raw""" +```julia +calculateR(A::QRhous{T}) +``` + +Computes the R factor of the QR factorization of `A`. +The R factor is the upper part of the matrix A, already transformed by the QR factorization. +The result is cached. + +### Input + +- `A` -- the QR factorization of `A`. + +### Output + +The R factor of the QR factorization of `A`. + +""" +function calculateR(A::QRhous{T}) where T + if !isnothing(A.AR) + return A.AR + end + m, n = size(A.QR) + min_size = min(m, n) + # @show triu(A.QR[1:n, :], 1) + A.AR = triu(A.QR[1:min_size, 1:min_size], 1) + diagm(A.d) + return A.AR +end + +@doc """ +```julia +(\\)(A::QRhous{T}, b::AbstractVector{T}) +``` + +Solves the linear system ``Ax = b`` using the QR factorization of A. +First, it computes the product ```Q^t b``` and then solves the triangular system ```R x = Q^t b``` via backsubstitution. + +### Input + +- `A` -- the QR factorization of A. +- `b` -- the right-hand side vector. + +### Output + +The solution vector x. + +""" +function (\)(A::QRhous{T}, b::AbstractVector{T}) where T + m, n = size(A) + v = qyhoust(A, b) + x = zeros(n) + tmp = x[1] + + for j ∈ min(m, n):-1:1 + @views tmp = dot(x[j+1:n], A.QR[j, j+1:n]) + x[j] = (v[j] - tmp) * inv(A.d[j]) + end + return x +end + +@doc raw""" +```julia +(*)(A::QRhous{T}, x::AbstractVecOrMat{T}) +``` + +Computes the product ``Ax,`` where ``A`` is the QR factorization of A. + +### Input + +- `A` -- the QR factorization of A. +- `x` -- the vector to be multiplied by ``A``. + +### Output + +The product ``Ax``. + +""" +function (*)(A::QRhous{T}, x::AbstractVecOrMat{T}) where T + return qyhous(A, multiplybyr(A, x)) +end + +@doc raw""" +```julia +show(io::IO, mime, A::QRhous{T}) +``` + +Pretty printing for the factoried matrix A. Computes explicitly Q and R. +""" +function Base.show(io::IO, mime::MIME{Symbol("text/plain")}, A::QRhous{T}) where T + summary(io, A) + println(io) + print(io, "Q factor: ") + show(io, mime, A.Q) + println(io) + print(io, "R factor: ") + show(io, mime, A.R) +end + +@doc raw""" +```julia +getproperty(A::QRhous{T}, name::Symbol) +``` + +Calculates explicitely Q and R if required. +""" +function Base.getproperty(A::QRhous{T}, name::Symbol) where T + if name === :R + return calculateR(A) + elseif name === :Q + return calculateQ(A) + else + getfield(A, name) + end +end + +@doc raw""" +```julia +propertynames(A::QRhous, [private::Bool = false]) +``` + +Returns a tuple of all the properties of a QRhous matrix. + +```jldoctest +julia> :R ∈ propertynames(qrfact([1 0; 0 1])) +true +``` +""" +Base.propertynames(A::QRhous, private::Bool=false) = (:R, :Q, (private ? fieldnames(typeof(A)) : ())...) + +@doc raw""" +```julia +size(A::QRhous) +``` + +Returns the size of the original matrix A. It is also the size of the support matrix. +""" +Base.size(A::QRhous) = size(getfield(A, :QR)) + +end # module housQR diff --git a/project/data_for_testing/dataset.csv b/project/data_for_testing/dataset.csv new file mode 100644 index 0000000..3180f4e --- /dev/null +++ b/project/data_for_testing/dataset.csv @@ -0,0 +1,1000 @@ +-0.9172796;-0.7127266;-0.9899035;0.9928187;0.9936488;0.995543;0.7110739;0.40764457;-0.68854785;0.6168897 +-0.8587842;0.9987552;-0.99839646;0.99990857;0.31650263;-0.9518967;-0.16313872;0.98098207;0.6617593;-0.8001547 +-0.9904415;0.9587257;-0.99867487;0.99721634;0.98716646;0.35648263;-0.2796892;0.5991635;-0.6846301;0.9229005 +0.9371171;0.9844736;-0.6124203;0.9998116;0.7286228;-0.53996205;-0.16593921;0.9993516;-0.92144436;-0.9747665 +-0.90662795;-0.88456744;-0.9324874;0.9410374;0.97813416;0.9981795;0.74960643;-0.5905986;-0.5082678;0.6917981 +0.9871297;0.9808353;0.20905054;0.9994791;0.2773453;-0.2684187;0.79519403;0.9995423;-0.6915291;-0.9982181 +-0.99377966;-0.822281;-0.81912893;-0.18078056;0.65813035;0.98782074;0.6454475;-0.95970315;0.82351744;0.7699433 +-0.88581;0.9889053;-0.99358505;0.9987541;0.9541387;-0.09235309;-0.8548569;0.97017556;-0.8389497;0.30218804 +0.7378768;-0.97811925;-0.6132943;0.93516594;0.9966666;0.99948925;-0.2740152;0.13181362;-0.95588106;0.7074977 +-0.96813554;0.85002303;-0.99782526;0.99954426;0.99403507;0.875412;0.60119843;0.9236852;-0.67161083;0.09033291 +-0.9571663;0.9911378;-0.99709064;0.9992754;0.96251756;-0.19512388;-0.5665988;0.9545374;-0.7234168;0.44910675 +-0.99110615;0.99602985;-0.9995716;0.9991432;-0.21076839;-0.96644783;-0.049464572;0.26849708;0.8553109;0.48096156 +-0.87714225;0.99769783;-0.9978437;0.99991304;0.90434223;-0.65752125;0.6915087;0.981739;0.30508164;-0.8037423 +0.31406438;-0.9820715;-0.7257031;0.95626676;0.99526244;0.9994798;0.29187787;-0.28284013;-0.913715;0.7439047 +0.12939906;-0.99159783;0.9781121;-0.9925454;0.259467;0.9489176;0.86351764;-0.9245098;0.9352836;-0.79464704 +0.9960535;0.80295837;0.9638762;0.8502266;-0.8969705;-0.71747494;-0.53535414;0.99636006;-0.4004438;-0.99931276 +0.8794734;-0.99282205;0.9671897;-0.8356731;0.9748471;0.9981283;-0.3740778;-0.6116591;-0.33136296;-0.08872179 +0.8917445;0.82180715;0.92574126;0.4669056;-0.9817921;-0.91625035;-0.2884787;0.9864289;0.8753733;-0.99973446 +-0.8584389;0.9969513;-0.99781704;0.9999084;0.9252936;-0.6862361;0.13922474;0.9891706;-0.11398312;-0.7744704 +-0.98474926;0.9851928;-0.99858934;0.99968135;0.9839416;0.0872131;0.20804846;0.91815186;-0.6287802;0.4189631 +0.42001423;-0.9904015;0.9177506;0.23517592;0.9873552;0.999656;0.6791664;-0.915095;-0.91883934;0.70935625 +0.9951222;-0.2154632;0.99280083;-0.0024526517;-0.9701035;-0.68467623;-0.17443453;0.96974194;-0.11036778;-0.9991679 +0.9757061;-0.91608256;0.9953574;-0.96945566;-0.9093659;0.53373504;0.5593062;0.91117084;0.3770278;-0.9975449 +-0.9924805;0.4636029;-0.99880844;0.9914531;0.98964083;0.9294616;0.39587107;0.15615726;-0.23417331;0.908096 +-0.7235204;0.9967639;-0.99701804;0.99992526;0.9469659;-0.57560277;0.32662565;0.99323267;-0.24631543;-0.8733967 +-0.8172529;0.9979425;-0.99673223;0.99990684;0.89935106;-0.60996234;0.7402701;0.98711556;0.31133792;-0.8711042 +0.99317205;0.8295896;0.77808803;0.9947163;0.1517892;0.74712336;0.041513912;0.9984513;-0.93641585;-0.987079 +-0.951579;0.98945564;-0.9967685;0.9995807;0.977918;0.007139098;-0.347872;0.9686852;-0.7475646;0.11524854 +-0.11387263;0.9983381;-0.9903618;0.99990803;0.30363187;-0.95857704;0.17051616;0.99719334;0.054537605;-0.956392 +0.08617923;-0.98935485;0.91387004;0.052771106;0.9871238;0.99955016;0.8500381;-0.9100371;-0.72887754;0.44358963 +0.7913781;-0.9847475;-0.3111227;0.8975178;0.9957505;0.9995402;-0.33835173;-0.15588924;-0.96865904;0.8014644 +-0.9781574;0.99726504;-0.9991432;0.99982643;0.7497177;-0.8476999;0.3694584;0.90913737;0.5751193;-0.023511216 +0.89985794;-0.97742444;0.961999;-0.9931664;0.22063288;0.9270748;0.1855414;0.09398908;0.80938375;-0.9778649 +-0.9918375;-0.696568;0.41767883;-0.7342258;-0.75509304;0.91258836;0.0027167785;-0.95976794;0.9889939;-0.6360037 +0.6286659;-0.9865661;0.9757443;-0.99404347;0.038044177;0.91577125;0.8028423;-0.66693234;0.9046736;-0.9282297 +-0.72787637;0.34656847;-0.9916285;0.9943776;0.9980402;0.9760233;-0.5026258;0.92935765;-0.9557106;0.6298807 +-0.9640983;0.98307735;-0.99710935;0.99927;0.98438215;0.20278344;-0.40835074;0.9431164;-0.80627495;0.4953873 +0.8281625;-0.9837021;0.9792315;-0.9939758;0.010386179;0.90790486;0.7845599;-0.24274269;0.8068356;-0.9673548 +-0.98344314;0.9963331;-0.99913365;0.9998586;0.9231848;-0.6069808;0.68172437;0.9288534;0.36786;-0.23665527 +0.75054806;0.9932692;-0.9187293;0.99992895;0.86786705;-0.39768663;0.58850443;0.9993089;-0.65423447;-0.98489124 +-0.40247867;-0.9862098;-0.42512223;0.68856543;0.9897087;0.9994231;0.22156791;-0.85884607;-0.9297664;0.9464222 +-0.98109055;0.9971051;-0.9991996;0.9997604;0.5362334;-0.9021473;0.17007579;0.86604273;0.6605973;0.18830016 +0.9973477;0.77271104;0.921333;0.98564345;-0.3816762;0.6986173;-0.21446967;0.9973221;-0.96873957;-0.9834281 +-0.5092116;-0.697627;-0.9869803;0.9977666;0.99731123;0.997151;0.67174995;0.8935489;-0.8110271;-0.30908316 +-0.9235607;-0.40976495;-0.99457526;0.987588;0.9975823;0.98941815;-0.1385373;0.60468477;-0.9132636;0.9019681 +0.61149246;0.9951097;-0.9565687;0.9999401;0.87953395;-0.52209866;0.46813744;0.99912935;-0.6267533;-0.9778796 +-0.97534204;0.099675834;-0.99524224;0.9986228;0.97741324;0.9693467;0.9262815;0.643563;-0.0017483962;0.1266657 +-0.99492264;0.9799816;-0.9993492;0.99724686;0.90615463;-0.59087473;0.005708532;-0.018920768;0.3703623;0.9489517 +-0.9720612;0.928546;-0.99795;0.9996922;0.99336547;0.76777464;0.5662266;0.932967;-0.635135;-0.01081398 +0.79674745;-0.99318033;0.97021407;-0.9861949;0.9198109;0.9922483;-0.16439159;-0.7377194;0.4353566;-0.58426803 +-0.9761635;0.99292535;-0.9982265;0.9995823;0.95515066;-0.39873153;-0.18992868;0.941039;-0.54929525;0.44990373 +0.36858532;-0.98812956;0.85640675;0.5278071;0.9887007;0.99965984;0.76471126;-0.87554896;-0.8923347;0.5888485 +0.70999926;0.9922664;-0.92949736;0.999838;0.81799126;-0.6917978;-0.8053706;0.9984077;-0.92403716;-0.9014846 +0.9873269;0.9585748;0.5297671;0.9992892;0.26836273;-0.10264768;0.08090642;0.9992218;-0.9561247;-0.986188 +0.99180365;0.33551887;0.9821732;0.54969126;-0.979294;-0.88172317;-0.119997516;0.98600453;-0.078451976;-0.9995591 +-0.9955082;0.08452629;-0.9765669;0.95954794;0.6889354;0.9381598;0.8843092;-0.4909277;0.8237362;0.054664217 +-0.99672765;0.96933144;-0.99955356;0.9983632;0.9289097;-0.45452502;0.60699177;0.0016710818;0.6462265;0.8987603 +-0.9969651;0.98742527;-0.999678;0.9975837;0.48297852;-0.9158396;0.16438487;-0.55738556;0.84686327;0.8809321 +-0.8199031;0.6941245;-0.9930519;0.9973777;0.99729055;0.9473325;-0.34662175;0.9506695;-0.94144994;0.43724418 +-0.99486476;0.95949465;-0.99925494;0.99072164;0.9282009;-0.38934395;-0.30676466;-0.43298274;0.3351558;0.9771901 +-0.35349065;-0.9875637;0.6161162;0.47964844;0.98708904;0.9996484;0.62716085;-0.9451602;-0.9250413;0.9151358 +-0.99514216;0.9923367;-0.99952984;0.9994911;0.7606088;-0.8286591;0.6151402;0.27496213;0.6234499;0.8212115 +-0.99144423;0.9958654;-0.9995591;0.9995328;0.21654317;-0.9406926;0.3244938;0.49485758;0.7900535;0.52635664 +0.5068629;0.99504167;-0.9654382;0.99994236;0.9063692;-0.4003336;0.5352436;0.9990248;-0.55852044;-0.97840345 +-0.91210896;0.9710783;-0.9967915;0.9998627;0.989038;0.6333604;0.68751395;0.9805589;-0.42595515;-0.7902293 +0.99294233;0.11785149;0.88161546;0.9004371;0.50055075;0.9773717;-0.5261066;0.99462706;-0.9573266;-0.9702478 +0.9959595;-0.56387484;0.9921082;-0.83816856;-0.86745703;0.46953797;-0.7570045;0.97532403;-0.024282793;-0.9985039 +0.1019247;-0.97016555;-0.86983436;0.9145483;0.99729127;0.9991158;-0.60563195;0.03529811;-0.9644658;0.8979918 +0.98312324;0.23799403;0.59550834;0.96102047;0.7248119;0.9789824;0.3740235;0.99686056;-0.77307093;-0.99256617 +-0.99514663;0.9718;-0.9992625;0.9996144;0.9686529;0.26469374;0.8414394;0.75406516;0.2058417;0.41030332 +-0.98981553;0.8298369;-0.9986284;0.99770796;0.9911005;0.83294374;0.1654871;0.7188689;-0.7349742;0.82387936 +-0.59743977;-0.82090867;-0.98725235;0.99649155;0.99767214;0.99772424;0.5786814;0.8017933;-0.8510161;0.12353145 +-0.8841043;0.9984519;-0.9984507;0.99991745;0.6634429;-0.90088093;0.21937819;0.9788296;0.5778227;-0.7576587 +0.8573888;-0.9599564;0.9417212;-0.9910329;-0.19043604;0.89873123;-0.018099459;0.023440238;0.9355804;-0.98621804 +-0.99833524;0.72580975;-0.998751;0.9906473;0.92284;0.57962257;0.66733557;-0.33103874;0.8614162;0.6470333 +0.9747998;-0.8833193;0.9914703;-0.97968394;-0.8501042;0.684096;-0.1508371;0.9379488;0.5519182;-0.99783677 +-0.9842784;0.9937521;-0.99888176;0.9998043;0.9571203;-0.43668583;0.39206076;0.9337631;-0.22519334;0.14716089 +-0.7555951;0.9927332;-0.99557674;0.99991655;0.97680956;-0.06580098;0.26900676;0.993817;-0.55770105;-0.8817938 +-0.8810144;0.9539772;-0.99322206;0.9989101;0.99190116;0.66881585;-0.57310224;0.9708417;-0.91291535;0.18861558 +0.14261931;0.98485297;-0.97764677;0.99990976;0.97411734;0.22902547;-0.027678626;0.998206;-0.8476759;-0.9432682 +-0.99632657;0.98818654;-0.9995893;0.9988246;0.75903946;-0.82807153;0.4958401;-0.097045064;0.6953653;0.8875566 +0.8902432;-0.9902462;0.94778806;-0.911581;0.9553161;0.9962183;-0.58706;-0.34972966;0.13260761;-0.5185248 +0.75796443;-0.9660439;0.9337664;-0.9916418;-0.017243475;0.91549563;0.03235908;-0.3924512;0.9559334;-0.97286487 +0.9704041;-0.9608475;0.95187455;-0.9885137;0.13819644;0.9500233;-0.5344328;0.7523884;0.72169656;-0.9888743 +0.99324906;0.8583723;0.8209873;0.9922021;0.21967857;0.57928646;-0.67009175;0.9981952;-0.9775684;-0.9634809 +-0.9954146;0.9776491;-0.9993334;0.9996369;0.9684919;0.15067367;0.81435955;0.7545168;0.16958658;0.46274137 +0.94479626;-0.8791065;0.98456633;-0.9864604;-0.8655158;0.69899344;-0.23750108;0.88142234;0.8560384;-0.99767554 +-0.9984388;0.8014634;-0.99937016;0.9916549;0.94970316;0.3799111;0.51427495;-0.45933673;0.8420501;0.77949417 +-0.9937459;0.6601376;-0.9975505;0.9983979;0.9539452;0.84210855;0.9002878;0.42955035;0.40804285;0.39915872 +0.97923845;0.80951047;0.93630356;0.7978255;-0.98074853;-0.9615906;-0.07877264;0.99415886;0.13458471;-0.9997491 +-0.9938987;0.9937271;-0.9995616;0.9993982;0.6058927;-0.8901838;0.4620814;0.29732093;0.6673507;0.7877544 +-0.751515;0.9195523;-0.9912313;0.99930197;0.99479944;0.84643996;-0.31936693;0.9864521;-0.92207694;-0.37295878 +-0.9971705;0.9739932;-0.99962616;0.99606127;0.8040679;-0.76841843;0.2648543;-0.54613566;0.7714937;0.9361752 +0.9802199;-0.87545514;0.99485344;-0.94294333;-0.9420271;0.29947063;0.49029905;0.9234393;0.40902084;-0.9982185 +-0.9864143;0.99401873;-0.99888754;0.99961066;0.90979683;-0.6708822;0.10188352;0.88170093;-0.09649938;0.56509787 +0.97119427;-0.991066;0.9868431;-0.9859439;0.9685185;0.99340016;-0.06504992;0.17550462;-0.5211759;-0.53168005 +0.8449787;-0.9835515;0.15276183;0.928951;0.9965535;0.99957097;0.5481052;0.006630316;-0.9185836;0.3095916 +-0.9966496;0.98563397;-0.99961466;0.99811095;0.73950183;-0.8347844;0.35535094;-0.30532548;0.736257;0.9068228 +0.9284373;-0.97394884;0.1866112;0.83100355;0.9956193;0.9994515;0.2797215;0.48548406;-0.8089776;-0.36963525 +0.46082997;0.9975529;-0.96351117;0.9999185;0.5712427;-0.9126011;0.59157443;0.99871004;-0.4131207;-0.9818617 +-0.91388357;0.99717814;-0.9981585;0.99990255;0.92476887;-0.55009073;0.7246132;0.97631854;0.29940587;-0.7749594 +-0.007175129;-0.98828477;0.7794682;0.53994876;0.98751944;0.99967766;0.7024751;-0.92487735;-0.9258402;0.8418757 +0.9326397;-0.99205977;0.97928447;-0.99198276;0.8959765;0.98195267;0.074302636;-0.20155528;0.1706112;-0.80021685 +-0.58771336;-0.32281202;-0.98851717;0.9941035;0.9987919;0.9919926;-0.3672092;0.91017634;-0.9663085;0.6448189 +-0.98603433;0.9938214;-0.99899954;0.99835527;0.75908947;-0.8463858;-0.4690821;0.6595827;0.34460488;0.8429561 +-0.9895054;0.8428016;-0.99859476;0.99779874;0.9911938;0.8195568;0.13760345;0.72668636;-0.7445353;0.82346284 +-0.9646069;0.98601234;-0.99780536;0.9998335;0.986269;0.2353881;0.3460506;0.9678481;-0.5330531;-0.37405124 +0.31074756;0.9866149;-0.9645425;0.9999127;0.96170986;0.38948503;0.63909143;0.9987682;-0.5763971;-0.9815646 +0.9675844;-0.98577714;0.7867564;0.26283336;0.992484;0.9994834;0.24744458;0.30480847;-0.878249;-0.2939299 +0.878542;-0.97233903;0.980747;-0.9926008;-0.45313874;0.86998355;0.64767325;0.2589108;0.8504026;-0.98676544 +0.9963994;-0.25684947;0.9952597;-0.027625503;-0.95287454;-0.5983239;-0.05007414;0.9661852;-0.39844447;-0.99866104 +-0.98869586;0.96805453;-0.99869627;0.9962412;0.9849323;0.16648664;-0.47824404;0.56378514;-0.6415536;0.941615 +-0.87669915;0.9944078;-0.996847;0.9998975;0.96869284;-0.30503502;0.023389244;0.99032885;-0.471645;-0.7623728 +-0.5995911;0.96234167;-0.99263555;0.9998134;0.9907145;0.7022437;0.29317567;0.9943442;-0.791588;-0.88860756 +-0.96160084;0.994619;-0.99796563;0.9997893;0.9563454;-0.40879348;-0.045661744;0.9723447;-0.4102084;-0.19446711 +0.9918725;-0.92525417;0.81544393;0.04427327;0.9419836;0.99788034;-0.35534623;0.9586122;-0.90417606;-0.9362078 +0.83089733;-0.9869206;0.9445538;-0.9855342;0.8243097;0.9834269;-0.30810052;-0.47675565;0.7562982;-0.8464529 +-0.7587702;0.9002499;-0.9922129;0.9993624;0.9953604;0.8840859;-0.13157994;0.9866236;-0.907546;-0.47852603 +0.96958643;-0.9332731;0.32325062;0.7256143;0.9857239;0.99875355;-0.23653725;0.9023517;-0.8067398;-0.8639568 +0.53540665;-0.993143;0.49619672;0.61945724;0.99167746;0.9995793;-0.11663302;-0.85904;-0.97964257;0.94260496 +0.8655568;-0.9636787;-0.39874384;0.936065;0.9969239;0.9994867;-0.25031647;0.6215019;-0.93450594;0.24887149 +-0.87539464;-0.01736615;-0.9953722;0.99802876;0.997836;0.98680776;0.35224572;0.89928114;-0.88290405;0.38376465 +0.009531595;0.9955749;-0.98471165;0.99993676;0.9451374;-0.1719478;0.638679;0.99810565;-0.33583844;-0.9729512 +-0.9904215;0.84108174;-0.99866074;0.99712676;0.99121517;0.81727964;0.054772127;0.66394013;-0.7350835;0.86541796 +-0.7851873;-0.99225;0.98865086;-0.98254454;0.8635909;0.99417526;0.83782953;-0.98240465;0.5751353;0.040287126 +0.9964891;-0.100025535;0.99475306;0.23806411;-0.94810426;-0.66222656;0.052591365;0.9713745;-0.51786;-0.9985705 +-0.9945301;0.9523444;-0.9990697;0.99948865;0.96640027;0.4607629;0.87130624;0.7282779;0.2849931;0.37950498 +-0.55031335;0.9879621;-0.99386555;0.99991006;0.98295075;0.38826665;0.5788247;0.9955314;-0.5299365;-0.94653076 +-0.66796494;-0.89825255;-0.9795361;0.9874918;0.997633;0.9980921;0.19796929;0.5085686;-0.9061348;0.73060197 +0.977989;-0.95579565;0.99471664;-0.99056524;-0.5956073;0.8627454;0.18681546;0.9148688;0.23993301;-0.99407697 +-0.86397743;-0.5597106;-0.9896538;0.9206758;0.9976265;0.98764884;-0.8059977;0.14104699;-0.955312;0.9790895 +0.9960186;0.30149505;0.91680527;0.9209104;0.13657308;0.95468956;-0.47573352;0.9953931;-0.96377796;-0.9732108 +0.8795172;-0.9866147;0.97245646;-0.994319;0.5301605;0.9441884;0.38250142;-0.19285676;0.69089705;-0.94705606 +-0.8958751;0.9708644;-0.9931683;0.9981568;0.98497456;0.46233898;-0.7916448;0.9559659;-0.90382916;0.48310763 +-0.83001405;0.74615157;-0.99316436;0.99733245;0.9969638;0.93298;-0.43158507;0.9509897;-0.94317305;0.46839312 +-0.92967206;0.99865353;-0.99778515;0.9999098;0.57595205;-0.88459444;0.790201;0.9700445;0.7698925;-0.81099063 +-0.97466266;-0.98053193;0.96558464;-0.99458086;0.44921404;0.9896015;0.7059482;-0.9953411;0.83585966;0.6974806 +-0.99666715;0.9859652;-0.99962527;0.99786067;0.69547546;-0.854412;0.27013955;-0.3719382;0.75746375;0.90598565 +-0.9214977;0.99125326;-0.99616385;0.99974245;0.9724299;-0.08850151;-0.35051775;0.98348975;-0.7069771;-0.33497545 +0.7729392;-0.9895928;0.067932226;0.7956198;0.9934302;0.9995219;-0.5041232;-0.5340059;-0.9801377;0.9014193 +-0.54900587;0.9872975;-0.9912044;0.99988335;0.9806731;0.13259426;-0.20448253;0.9958137;-0.81675637;-0.86451036 +-0.86320245;0.99756104;-0.99738604;0.99990284;0.9183209;-0.5317619;0.7515174;0.983462;0.30133843;-0.84869486 +0.0065886895;0.94780177;-0.97462195;0.99963874;0.98798186;0.7312358;-0.39191973;0.9968787;-0.9328508;-0.8576663 +-0.5827501;-0.9526992;-0.94645727;0.9410953;0.99636096;0.99862915;-0.18634601;-0.12327091;-0.9296463;0.92092294 +-0.99175864;0.9420695;-0.9987054;0.9980237;0.9884922;0.5400663;-0.03489393;0.65473056;-0.69485706;0.8842936 +-0.9950454;0.99269885;-0.99947846;0.9996613;0.79932344;-0.79333097;0.7260543;0.37744412;0.60571545;0.79168254 +-0.9942027;0.9936619;-0.9995274;0.99957526;0.694543;-0.8580667;0.61551684;0.36509082;0.63416064;0.7771123 +-0.9977663;0.92509043;-0.999497;0.9976756;0.95666426;0.13752341;0.6870323;-0.0834153;0.7544949;0.76447564 +-0.89523375;0.9395905;-0.9958263;0.9997296;0.99401176;0.8123743;0.3727252;0.97942984;-0.75192815;-0.5711934 +0.8492912;-0.9913997;0.82193303;0.69464415;0.99274;0.999618;0.27331382;-0.6779644;-0.97515416;0.7140231 +-0.71286476;-0.9952508;0.9914588;-0.99418634;0.90974796;0.996174;0.79705715;-0.99195045;0.053228773;0.6456495 +0.98622835;-0.83637166;0.9953893;-0.90443367;-0.95237726;0.10737925;0.418553;0.9346973;0.29059964;-0.99843025 +-0.9135794;0.9860357;-0.99450004;0.9983057;0.96572;0.06041485;-0.835314;0.9553161;-0.8445465;0.50115156 +-0.31402758;-0.9960939;0.9936239;-0.98125684;0.92976505;0.99512017;0.89525324;-0.97261864;-0.00037573275;0.026001545 +0.77973104;-0.9892387;0.0492665;0.8112548;0.9937046;0.99953014;-0.47935033;-0.49888858;-0.97955036;0.8930887 +0.9404686;-0.903754;0.98336333;-0.98798513;-0.8318917;0.7528699;-0.18950363;0.85268843;0.85477716;-0.99716717 +-0.994565;0.80826217;-0.999087;0.9956259;0.9887191;0.8484226;0.37343433;0.4325153;-0.333592;0.8794169 +-0.98449695;0.99454314;-0.9989237;0.9987746;0.7583011;-0.8490902;-0.4012764;0.7610462;0.30559644;0.77642196 +-0.99712884;0.96633315;-0.9996095;0.996275;0.87611485;-0.6455294;0.38424104;-0.4275411;0.736811;0.9361171 +0.9899026;-0.7488732;0.9923501;-0.90151644;-0.9197475;0.31068364;-0.34791592;0.9578171;0.28694338;-0.998766 +-0.80872566;0.9951057;-0.9969388;0.9999134;0.96607196;-0.19211619;0.5998773;0.9901137;-0.17842802;-0.8906128 +-0.9948033;0.94297826;-0.9991142;0.99885243;0.98539114;0.5572741;0.4509106;0.67579955;-0.44312662;0.7869064 +0.87770146;-0.98991996;0.93785834;-0.8655788;0.9558124;0.99674314;-0.6099155;-0.38929486;0.17166765;-0.50139505 +0.911269;-0.9868781;0.36191654;0.8684989;0.9957722;0.999587;-0.16769388;-0.046901464;-0.9743855;0.713854 +-0.97728944;0.6887746;-0.99777085;0.99113196;0.9944063;0.9003879;-0.4499435;0.63637304;-0.8652368;0.92575496 +-0.7295631;-0.9587462;0.124509886;0.8085615;0.97188544;0.99941003;0.9190023;-0.8444756;-0.21956176;0.12481481 +-0.85361814;0.98464;-0.99255025;0.99933124;0.9762768;0.1696201;-0.73216814;0.9829851;-0.866626;-0.10413392 +-0.88024133;-0.9540518;-0.7877443;0.7196906;0.97729635;0.9989076;0.55166644;-0.86813915;-0.6695854;0.88812834 +-0.7555676;0.9968324;-0.99724007;0.99992275;0.9419899;-0.6098187;0.2989656;0.9925702;-0.21267515;-0.85808814 +0.9863311;-0.7608939;0.9876838;-0.9641129;-0.75036806;0.76171696;-0.7199424;0.9663757;0.44024244;-0.9983543 +0.9483867;-0.9822568;0.95169574;-0.96483725;0.9222814;0.9924564;-0.5541138;0.24023385;0.2601842;-0.8120898 +-0.18402374;-0.99022883;0.9104996;-0.05190078;0.9868034;0.9995983;0.8427106;-0.9420329;-0.80923253;0.70351136 +-0.8669654;0.91376233;-0.9928948;0.99785954;0.99371696;0.7876849;-0.66758066;0.9563299;-0.93766844;0.4880958 +-0.9931425;-0.89770335;0.77861893;-0.9314195;-0.5662436;0.9670346;0.26345283;-0.98178387;0.97937465;-0.029492985 +0.99099976;-0.8227326;0.99502015;-0.93233895;-0.88964313;0.46867895;-0.18049644;0.9575613;0.022422101;-0.9981787 +0.98774105;0.21171276;0.6930789;0.95126253;0.63874227;0.97660494;0.3564564;0.9968576;-0.785519;-0.99304426 +-0.9633955;0.9940524;-0.99792504;0.99977505;0.96107095;-0.35495737;-0.0501028;0.97050184;-0.46725667;-0.12529711 +-0.99346113;0.94206464;-0.99904853;0.99941033;0.98533344;0.5778715;0.6209806;0.7926774;-0.43303356;0.58977157 +0.9743307;-0.9417894;0.99330676;-0.9897206;-0.71127236;0.8244936;0.061868273;0.91720647;0.4387328;-0.9956846 +-0.6178274;-0.095969;-0.9894639;0.9936894;0.9986896;0.98883516;-0.4605886;0.9182128;-0.96502244;0.65466094 +0.99074256;0.8574897;0.75892985;0.993957;0.40457374;0.66169465;-0.5986104;0.9984647;-0.97318274;-0.9658834 +-0.989075;0.9428654;-0.99867296;0.9993396;0.98959196;0.5848634;0.30108142;0.845073;-0.6798947;0.628281 +-0.96098864;0.99672234;-0.99794006;0.9993759;0.7464609;-0.82698077;-0.6191388;0.9438115;0.019449463;0.27261692 +0.9983331;0.018747767;0.99539864;0.16083059;-0.8630602;-0.2189912;-0.33985302;0.9745535;-0.699306;-0.9971523 +-0.9773445;0.9962722;-0.99865395;0.99930865;0.69020146;-0.8735976;-0.39240327;0.8854831;0.3387573;0.5192185 +-0.9665054;-0.8738009;-0.8306798;0.74574095;0.92866504;0.9977394;0.75579184;-0.8885194;-0.028231597;0.75195605 +-0.6967956;0.99841094;-0.99630886;0.9999238;0.76422006;-0.8645124;0.43588516;0.9918397;0.29863784;-0.88524187 +0.9968454;-0.3017186;0.9963233;-0.07673269;-0.9389604;-0.53557795;0.061567597;0.9633258;-0.520271;-0.99822736 +-0.97682226;0.993366;-0.998295;0.9996179;0.9511529;-0.44315585;-0.14199367;0.9416624;-0.5039338;0.41731378 +0.26751915;-0.994195;0.9833127;-0.9789493;0.93632746;0.99588;0.33352965;-0.94750434;0.47136337;-0.3057221 +0.86717004;0.9738819;-0.72215736;0.99979603;0.8878359;0.064701155;-0.14861272;0.99923867;-0.91705096;-0.9701714 +-0.91444373;0.9980843;-0.9985547;0.99991554;0.8061947;-0.8338775;0.4535443;0.97427285;0.4912845;-0.6898649 +-0.98047876;0.9638579;-0.9983005;0.99970675;0.991261;0.55383337;0.42697424;0.9256411;-0.6400254;0.16210811 +-0.96611166;0.91512144;-0.9978329;0.9997131;0.9936783;0.8087601;0.6169545;0.9397648;-0.6166489;-0.12751892 +-0.9927211;-0.081660986;-0.99226713;0.98780626;0.9486529;0.97255445;0.81142366;-0.17473163;0.34937525;0.6102857 +0.9887899;-0.8773909;0.99346584;-0.9812577;-0.62577784;0.84604496;-0.57010293;0.962833;0.02959899;-0.9967623 +-0.9702725;0.9953407;-0.9980782;0.99935555;0.8636884;-0.7294313;-0.5068623;0.9349758;-0.24096121;0.48524255 +-0.98340684;0.9907989;-0.99866146;0.99789846;0.9161742;-0.62132436;-0.59768045;0.75889575;-0.23209266;0.87432754 +0.9303705;-0.98475325;0.36686534;0.8649262;0.99616855;0.99956894;-0.12130267;0.15918213;-0.96316856;0.5503473 +-0.60302955;0.9912961;-0.9875761;0.99967664;0.93002015;-0.3238539;-0.89468575;0.99291986;-0.871145;-0.5845758 +-0.94397193;0.9760505;-0.9964443;0.9996616;0.9911079;0.4969099;-0.027646372;0.97117597;-0.78212523;-0.15953708 +-0.9955117;0.97296804;-0.99921054;0.9982475;0.9648766;-0.14393957;0.3592581;0.30254045;0.011247548;0.9233786 +-0.99817777;0.47804326;-0.9963768;0.9664326;0.83047676;0.734934;0.68253225;-0.5086759;0.8991448;0.4967996 +-0.98203987;0.994964;-0.9987203;0.999715;0.9183253;-0.645299;0.13302316;0.9294946;-0.12731907;0.29609185 +-0.2384227;-0.9803593;0.7995272;-0.85939944;0.6891354;0.99150026;-0.58597416;-0.8678642;0.9505129;-0.7365425 +-0.41883034;0.99638253;-0.9938918;0.9999375;0.9209985;-0.70859784;-0.37194443;0.996891;-0.6565098;-0.8757963 +-0.9916163;0.9882033;-0.9990845;0.9996936;0.97074383;-0.20325941;0.41776305;0.84654504;-0.39341155;0.61403346 +0.75473946;-0.97292835;0.87457114;-0.97292244;0.59523773;0.9779716;-0.5697877;-0.4589856;0.93589604;-0.9305698 +0.98818195;0.18758659;0.9770977;0.20695911;-0.9843562;-0.84446824;-0.37735164;0.9807746;0.42871672;-0.9996749 +-0.48522678;-0.99410015;0.98629;-0.981792;0.8299418;0.9902513;0.8519168;-0.97031873;0.65087485;-0.41996637 +0.9919152;0.7415747;0.78942555;0.98834056;0.4127144;0.8534998;-0.44846463;0.9980771;-0.9616746;-0.9735993 +-0.995973;0.15554152;-0.9971297;0.9825944;0.9613858;0.93515575;0.5861202;-0.27806646;0.4984686;0.80479306 +0.9910632;-0.69856465;0.9947145;-0.7388234;-0.96882725;-0.2946597;0.17623587;0.9493087;0.23732778;-0.99892527 +-0.9985945;0.65561754;-0.99920046;0.9724626;0.92611754;0.40915328;0.35007215;-0.67320293;0.90153164;0.7995105 +-0.9941059;0.9892914;-0.9992523;0.9994913;0.93778914;-0.5099923;0.44052598;0.6255355;0.055279214;0.8134128 +-0.88140976;0.96004665;-0.9956525;0.9997763;0.9923915;0.71752214;0.32333916;0.98434585;-0.7140271;-0.69025624 +-0.845218;0.9924693;-0.9929747;0.9992514;0.92237073;-0.35612047;-0.8875432;0.9811692;-0.81441104;-0.0312824 +0.19966023;-0.99096334;0.535124;0.7558108;0.99184656;0.9996553;0.5137646;-0.87517244;-0.9614793;0.89221704 +-0.30512315;0.99584246;-0.9928994;0.9999444;0.9434031;-0.6061103;-0.2399534;0.9973417;-0.6947715;-0.9043355 +-0.7881008;0.7379291;-0.9923526;0.99785346;0.9973072;0.9445144;-0.3374365;0.9628363;-0.943021;0.2803334 +-0.9834214;0.9950193;-0.9988458;0.999288;0.7983195;-0.824468;-0.2378872;0.84901327;0.21391818;0.65062207 +0.9032371;0.9584152;-0.45945746;0.9994777;0.8721042;0.17285499;-0.75787246;0.99857664;-0.97312665;-0.9239297 +0.6902433;-0.9889669;0.7091838;0.8060261;0.99341804;0.9996566;0.7346958;-0.6787983;-0.9257131;0.4751084 +-0.94675404;0.97777617;-0.99657357;0.99967164;0.9906468;0.46328753;-0.022520399;0.9706516;-0.77212906;-0.15299009 +-0.96678245;0.6582024;-0.99730736;0.99500626;0.9949937;0.9222338;-0.2988905;0.78932506;-0.8840151;0.8488419 +0.97051495;-0.9625066;0.32591102;0.66633165;0.9897475;0.9990406;-0.05333177;0.8328794;-0.7783577;-0.766347 +-0.9796301;0.97022843;-0.9980744;0.9962029;0.9848939;0.20231187;-0.66502327;0.7383083;-0.7602857;0.9154347 +-0.9946291;0.99325836;-0.99957603;0.9990128;0.37081617;-0.92716676;0.32637435;-0.0070567;0.7453706;0.81019354 +0.71882236;-0.9956403;0.9921423;-0.99523646;0.647882;0.96879697;0.9113216;-0.7894885;0.28075802;-0.75519484 +-0.6049591;-0.97482216;-0.32440025;0.85420305;0.98583096;0.9994713;0.8069619;-0.83286923;-0.813654;0.74652445 +0.93856966;-0.99568075;0.9920959;-0.9951925;0.9596046;0.9916188;0.29342914;-0.5896761;-0.50807416;-0.33836332 +-0.99299324;0.9857234;-0.99906117;0.9995249;0.97103584;-0.1694051;0.3684026;0.7691025;-0.39016393;0.75912696 +0.9976382;0.26984447;0.98967385;0.34427103;-0.92164916;-0.4396105;-0.5765517;0.9839306;-0.4052658;-0.99877626 +0.9842226;-0.9263566;0.99063027;-0.9886872;-0.28400028;0.9287052;-0.4904753;0.94415843;0.1645915;-0.9941857 +-0.994075;0.6697348;-0.99901605;0.9886644;0.9900756;0.8928779;0.1688452;0.105779454;-0.27215686;0.9387195 +-0.92646056;0.21368691;-0.99623746;0.99651194;0.9973671;0.9777272;0.010878331;0.84902626;-0.895507;0.6902076 +-0.8273793;0.97728306;-0.9919127;0.99937844;0.9853223;0.40923685;-0.6559966;0.9851885;-0.8894164;-0.21426094 +-0.55609214;-0.9711855;-0.81260544;0.88327426;0.9896778;0.9992242;0.3623605;-0.653609;-0.85576105;0.87558526 +0.9055069;0.9810728;-0.6558454;0.9997506;0.81411785;-0.40816;-0.62088895;0.999059;-0.94717896;-0.9537629 +-0.4227449;0.9971633;-0.994418;0.99994;0.93196607;-0.6372859;0.27765208;0.9964889;-0.3766192;-0.91989607 +-0.99528706;0.9924949;-0.9995908;0.9989248;0.44553456;-0.91953385;0.33933467;-0.08669978;0.746722;0.8330699 +-0.9959889;0.9856351;-0.9994719;0.9994112;0.90426755;-0.61696637;0.72879195;0.28477484;0.6037054;0.84775627 +-0.9241125;0.9691976;-0.9945772;0.99751425;0.98446316;0.43175107;-0.8099802;0.92868274;-0.8941214;0.69281566 +0.15584262;-0.99316585;0.08326144;0.45428994;0.9909813;0.9994503;-0.33274305;-0.9002236;-0.9734009;0.97199994 +0.83205664;0.9775978;-0.77876526;0.9998166;0.89869225;-0.006156026;-0.26951197;0.9991248;-0.92009014;-0.9656075 +0.9933874;0.9259261;0.73520064;0.99822956;-0.0020825022;0.35679805;0.44141415;0.9990865;-0.9236442;-0.99358827 +-0.9856164;0.9935798;-0.9989707;0.99982125;0.95901626;-0.41198385;0.4719114;0.92956954;-0.18023092;0.11930739 +0.7270433;0.9936372;-0.9337106;0.9999061;0.842716;-0.68879235;-0.38205105;0.9990296;-0.85126346;-0.955829 +-0.95272183;-0.67521715;-0.9915486;0.97443044;0.9931932;0.9930052;0.26792717;0.032961924;-0.74688214;0.9071951 +-0.797033;-0.94681895;-0.77639014;0.8962949;0.9797547;0.9991074;0.7836105;-0.7489998;-0.62099797;0.6650918 +-0.9603707;0.96818167;-0.9970264;0.9995111;0.99188167;0.5437912;-0.05169025;0.9535978;-0.80470735;0.18676667 +0.9276937;-0.9794984;0.19045483;0.87242687;0.9965167;0.99956447;-0.19683169;0.3539865;-0.9511744;0.41187936 +-0.834403;0.99319065;-0.99637395;0.99990577;0.9755832;-0.12959464;0.18924986;0.9919741;-0.50951296;-0.8360793 +0.8935527;-0.99202436;0.9725425;-0.98931694;0.9124699;0.9877825;-0.08427767;-0.46930432;0.34972006;-0.71548337 +-0.9902647;-0.82398033;0.7463335;-0.8925185;-0.68403536;0.93886757;0.16309913;-0.97438365;0.9902031;-0.5834262 +-0.8387938;-0.1142301;-0.9924882;0.98404354;0.9981354;0.9831191;-0.6445496;0.7703559;-0.9578362;0.9099769 +0.72728974;0.9685458;-0.8425985;0.9997492;0.9415421;0.26851845;-0.69633305;0.9983734;-0.96317333;-0.8978396 +-0.9234206;0.9818843;-0.9946035;0.9976646;0.972259;0.18521382;-0.8438417;0.9387023;-0.86117554;0.64084697 +-0.93008494;-0.34642535;-0.9949955;0.9869133;0.9975592;0.9879462;-0.19534767;0.59430194;-0.9145337;0.9125185 +-0.676967;0.95240724;-0.99038965;0.9995488;0.99188626;0.73098683;-0.36216012;0.99168473;-0.90111023;-0.6360959 +0.9181208;-0.980757;0.12412038;0.88663214;0.9965275;0.99957055;-0.19323266;0.2907303;-0.9564846;0.505206 +0.9810311;-0.9822266;0.9842063;-0.9938563;0.5097701;0.96100336;-0.1704207;0.7433508;0.1403089;-0.97270215 +-0.90590656;0.3944241;-0.99603516;0.9981253;0.99728644;0.973666;0.1963414;0.90996605;-0.87951607;0.41652778 +0.8417841;-0.9619718;-0.35660622;0.9430569;0.9966105;0.9994047;0.34334058;0.5455539;-0.823343;-0.2486138 +-0.77003205;-0.8625065;-0.9826455;0.9961685;0.994054;0.99792945;0.8513791;0.4278857;-0.53849494;0.03330191 +0.9871588;0.07800113;0.8044536;0.9247924;0.7162233;0.98376656;-0.55868155;0.994119;-0.95477164;-0.96550506 +-0.99045146;0.98916036;-0.9990254;0.99969727;0.97016186;-0.23478632;0.35805216;0.8662294;-0.42417112;0.5880252 +0.9657946;-0.983533;0.9752909;-0.9927315;0.6459852;0.9642751;-0.11512549;0.5069625;0.36343065;-0.9620259 +-0.4432022;-0.98448676;0.3368626;0.6837852;0.9874616;0.9996118;0.7127039;-0.916539;-0.90567017;0.8785484 +0.9571141;-0.71979743;0.9868366;-0.9072266;-0.97580355;-0.10316541;0.30411273;0.9083179;0.85247964;-0.9991128 +-0.9552982;0.98789126;-0.99679375;0.9994885;0.98042446;0.08326401;-0.37467903;0.96263003;-0.77192044;0.2385144 +-0.98550695;0.9963305;-0.99917966;0.9996647;0.65660214;-0.8810553;0.15857291;0.8332832;0.56023437;0.4251984 +0.89028496;-0.99273753;0.9651595;-0.27156064;0.98642015;0.99962634;0.2689965;-0.8353031;-0.95415914;0.60949504 +0.17732398;0.9773413;-0.9741229;0.99988353;0.9780061;0.48328358;0.15438288;0.99832433;-0.82191336;-0.95491815 +-0.9077188;-0.9074075;-0.9186467;0.9019998;0.97807175;0.9983391;0.66363317;-0.69287395;-0.5590746;0.79279315 +-0.5824482;-0.89732563;-0.9819622;0.9937779;0.99755305;0.9982755;0.487971;0.6357687;-0.868306;0.42482924 +-0.8329084;0.9310889;-0.9916732;0.9987153;0.9935759;0.77674913;-0.5851683;0.9743226;-0.9343045;0.14994732 +-0.9569628;0.98492116;-0.9968754;0.9995883;0.98610574;0.23190275;-0.1972608;0.9642115;-0.76975477;0.10070537 +-0.90395266;0.98133796;-0.9937276;0.99857295;0.9785435;0.2703226;-0.78539246;0.9622369;-0.87502813;0.39415017 +0.90235853;-0.99129426;0.9589242;-0.9400007;0.96042824;0.99593514;-0.5088546;-0.35072324;-0.004699762;-0.47350487 +-0.99670464;0.9785659;-0.9995656;0.9982258;0.8712361;-0.690793;0.5256041;-0.16716234;0.6880035;0.9159744 +0.4995273;0.99402547;-0.9672863;0.99993867;0.91861236;-0.5091758;-0.10813591;0.9989127;-0.7843458;-0.960871 +0.99698865;0.4791882;0.9831808;0.5132075;-0.93269444;-0.5790424;-0.6167866;0.9895139;-0.29719576;-0.9991737 +-0.7929996;0.9801036;-0.9904461;0.99928933;0.97968817;0.30670598;-0.76231384;0.98623806;-0.8960167;-0.20494324 +0.9620579;-0.98333657;0.6404891;0.5213427;0.99361354;0.99948776;0.26882342;0.38446862;-0.86033595;-0.37794366 +-0.99529326;0.98614436;-0.9994138;0.99972737;0.9593702;-0.10944616;0.8080158;0.76716596;0.24147673;0.45747498 +-0.99733853;0.44058597;-0.9989392;0.9631174;0.9773269;0.8438829;0.1051698;-0.56337845;0.5350585;0.9379907 +-0.993682;0.9887875;-0.9994267;0.9985324;0.8409476;-0.7628219;0.0029072922;0.2491496;0.48829004;0.9024887 +-0.87259245;-0.9918901;0.9862548;-0.99317485;0.838377;0.994625;0.7834127;-0.9917575;0.49338013;0.5364682 +-0.9945313;0.98651475;-0.9992126;0.9993173;0.94963336;-0.4127057;0.40132123;0.56972075;-0.010529174;0.8545238 +0.99366945;-0.6307689;0.9918615;-0.7874063;-0.9436967;0.07269325;-0.5055425;0.963917;0.21532027;-0.9990372 +0.029372945;0.97907627;-0.9812129;0.99988467;0.9813554;0.46304977;0.116182506;0.99796104;-0.8252825;-0.9451689 +0.9530321;-0.99003345;0.9768261;-0.974229;0.9642637;0.9942876;-0.2934278;0.032216772;-0.27883947;-0.5555358 +0.41452843;-0.9893773;-0.2666545;0.89188385;0.99336654;0.99954647;0.07749779;-0.6396619;-0.96070397;0.8885871 +-0.98984075;0.7988415;-0.99861425;0.9965871;0.99160594;0.856686;0.050509118;0.6543668;-0.7457743;0.8703226 +0.8100522;-0.9940555;0.9848967;-0.95241666;0.9796771;0.9979996;0.06925698;-0.831275;-0.3602246;0.085185856 +-0.9664195;-0.33453158;-0.99580854;0.9901952;0.99476266;0.9873313;0.44705284;0.41258007;-0.7296706;0.8521646 +0.79786515;-0.9974316;0.9970543;-0.9959515;0.84966457;0.9853027;0.91909945;-0.7917276;-0.47522664;-0.47113898 +-0.6236832;-0.93159574;-0.96617717;0.9743562;0.9970626;0.99842316;0.032782365;0.2240191;-0.9209245;0.85127157 +-0.84712535;0.9587325;-0.9922481;0.99913424;0.9915828;0.65847373;-0.57181704;0.97908443;-0.9129304;-0.018331628 +-0.9766579;0.97948706;-0.99802125;0.9966515;0.97682446;0.013337568;-0.72030354;0.7884606;-0.7176809;0.8950356 +0.05379582;0.85670155;-0.97131;0.99917465;0.9922439;0.91292024;-0.2618676;0.9964655;-0.9423845;-0.8475548 +0.87232167;-0.14439279;-0.3167278;0.9813789;0.95380145;0.99256307;0.39776585;0.9936488;-0.6668586;-0.98673147 +-0.9702791;0.9985471;-0.9992432;0.99988467;0.34871677;-0.9329598;0.41341138;0.92381775;0.8362055;-0.5348197 +-0.9882261;0.9797467;-0.99880683;0.99666137;0.973449;-0.16492155;-0.53232825;0.59676564;-0.48050824;0.9342451 +-0.9933657;0.95356023;-0.9990179;0.99111414;0.97612244;0.11940117;-0.40438086;-0.041750874;-0.32569072;0.97685987 +0.997196;0.5195266;0.9744324;0.59855634;-0.8797582;0.33448213;-0.7563481;0.99312186;0.49188974;-0.99880403 +-0.99478656;0.9822621;-0.99943596;0.99653953;0.83261895;-0.7296439;-0.19944677;-0.19704449;0.5649782;0.9484201 +0.9763426;-0.9308103;0.9937275;-0.9879846;-0.7537628;0.79649264;0.02386924;0.92816645;0.4034329;-0.99633414 +0.9962525;0.5990932;0.8698946;0.9800999;-0.10793621;0.88101107;0.22322921;0.99728674;-0.90987;-0.9913894 +-0.9559419;0.9772848;-0.99689406;0.99963534;0.99051154;0.44999018;-0.020297663;0.96433234;-0.77257705;-0.018201776 +0.95519257;-0.97919405;0.96182525;-0.99091506;0.5928685;0.96402025;-0.25463057;0.44545087;0.5977987;-0.9675706 +-0.98498785;0.9977183;-0.99935985;0.99985844;0.4090734;-0.91597337;0.5828622;0.8334305;0.8059741;-0.050073147 +0.99787766;-0.014137461;0.99687845;0.4113987;-0.86450547;-0.5207051;0.46667615;0.972751;-0.8404369;-0.99600095 +0.91414374;-0.9811237;0.9141654;-0.86856383;0.93084985;0.9955021;-0.6838661;0.04681466;0.39375675;-0.7572806 +0.90508384;-0.96533823;0.9848573;-0.99034095;-0.70814085;0.8275256;0.5785522;0.55854756;0.843637;-0.9925091 +-0.82146174;-0.8443146;-0.9825335;0.97256;0.9969854;0.9965949;-0.10482983;0.2909803;-0.9197717;0.9105483 +-0.99704295;0.5867718;-0.99315053;0.9911941;0.79807127;0.7908587;0.883235;-0.19280444;0.8378477;0.07511579 +-0.3282338;-0.4426482;-0.98443496;0.9955691;0.9987505;0.9954365;-0.14557454;0.94815314;-0.95822483;0.17491947 +-0.9708743;-0.007274082;-0.9972903;0.9965613;0.9947696;0.9817117;0.64607733;0.69024295;-0.6649156;0.6705141 +-0.8536935;0.7846125;-0.99338394;0.9956123;0.99595934;0.9003492;-0.67587936;0.92937803;-0.9490052;0.703569 +0.977799;0.9826846;-0.096937336;0.9997485;0.19134168;-0.76895225;0.25566077;0.99941564;-0.93263584;-0.9897826 +-0.9858443;0.9880059;-0.99892104;0.9998569;0.9551054;0.044508617;0.8748899;0.90446377;0.40755376;-0.51844406 +-0.9913425;0.99397945;-0.99928117;0.9996255;0.85321873;-0.7630631;0.3556012;0.72678995;0.29549542;0.6927925 +0.5708082;-0.9875928;0.44677928;0.9032574;0.9943935;0.9996491;0.8097474;-0.5955058;-0.9097995;0.43563712 +-0.9939804;0.9412381;-0.999086;0.9993356;0.9848308;0.57770413;0.6198588;0.77268004;-0.40634295;0.6258401 +-0.99712306;0.6442877;-0.9942885;0.9936838;0.82309884;0.7610859;0.8895266;-0.12786956;0.82727534;0.088149644 +0.6073749;-0.97134036;-0.7791909;0.9522818;0.9972247;0.9994255;-0.26746845;0.29522014;-0.949423;0.66973466 +-0.59084296;0.9729625;-0.99252814;0.99984235;0.98946303;0.5976013;0.20843774;0.99490005;-0.7922956;-0.89032954 +-0.9708854;0.9893437;-0.9978496;0.99614835;0.9128151;-0.51859903;-0.8637094;0.77808654;-0.48364606;0.89559776 +-0.38070682;-0.9958446;0.99279326;-0.9830359;0.91092587;0.9942279;0.8959921;-0.97344995;0.15749967;-0.05747235 +-0.9176309;0.9980003;-0.9985627;0.9999145;0.82320005;-0.8214354;0.46576384;0.973994;0.46351117;-0.6794319 +-0.4083177;-0.986088;-0.4248997;0.68926394;0.9896617;0.9994237;0.23221341;-0.8592739;-0.92897284;0.9458678 +-0.77448714;0.5850772;-0.99184763;0.99536;0.9975071;0.95692515;-0.54796344;0.93770915;-0.9567836;0.6304294 +-0.8126714;0.9642899;-0.99163216;0.99931;0.9905813;0.6167475;-0.5837133;0.98435366;-0.9090399;-0.19366664 +0.96440554;-0.98716116;0.98068;-0.9930802;0.74410146;0.9692807;-0.0024075834;0.4276692;0.1830683;-0.9408658 +0.9245909;-0.97337866;0.8717963;-0.8437855;0.90060234;0.9938985;-0.7037723;0.27307123;0.5396701;-0.86382365 +0.348332;0.99775356;-0.97522974;0.99992;0.50925875;-0.93152267;0.47830278;0.9985261;-0.34207034;-0.9762652 +-0.60132045;-0.96299976;-0.8678107;0.9197429;0.9899614;0.99912244;0.4336734;-0.5560194;-0.8271916;0.83609605 +-0.38796076;-0.9271327;-0.9744688;0.99316704;0.99720114;0.998758;0.56620705;0.54900235;-0.82525986;0.28820783 +-0.9951509;0.9923872;-0.99951273;0.9995619;0.7842102;-0.8114108;0.65683645;0.31708208;0.6138508;0.81239533 +-0.7521103;-0.9813798;0.62861127;0.18670233;0.97361696;0.99956095;0.7964307;-0.95321524;-0.6969138;0.831464 +-0.99651635;0.9152583;-0.9993319;0.9956515;0.9810935;0.500844;0.40619266;0.09861451;0.2100944;0.9075957 +-0.8365228;0.9272647;-0.9949422;0.9997269;0.99437886;0.8457593;0.43825662;0.9842676;-0.7647431;-0.6933022 +-0.8635961;0.9954137;-0.99737173;0.99990916;0.9624512;-0.29825854;0.54378533;0.9880877;-0.1651247;-0.84744596 +-0.9970628;0.9533657;-0.99960107;0.9970021;0.9369398;-0.34197032;0.50732446;-0.20052819;0.6823579;0.9181901 +0.998737;0.47147465;0.9898684;0.6437626;-0.84628063;0.20362495;-0.72854924;0.9920172;-0.2751559;-0.9977286 +0.75101376;-0.98799145;0.6154772;0.8142889;0.9940483;0.9996238;0.76541936;-0.527171;-0.8782637;0.20475288 +0.9969889;0.36042055;0.99321836;0.7151363;-0.9124965;-0.7758262;0.47614494;0.98454934;-0.812249;-0.9978325 +-0.8285814;0.80580634;-0.9925164;0.99650455;0.9960273;0.8992245;-0.65111685;0.94547164;-0.95159525;0.60816276 +0.10742208;-0.98894066;0.14584705;0.8773244;0.99236625;0.9996268;0.6458379;-0.78970367;-0.94616556;0.83983713 +-0.9118176;-0.85754865;-0.94904435;0.96210015;0.9794348;0.9978704;0.78508574;-0.4689151;-0.48347893;0.6247089 +0.9023782;-0.9782844;0.9838007;-0.9928927;-0.30406606;0.8908954;0.67984647;0.31326094;0.7597241;-0.9843249 +0.47629112;-0.98447263;-0.57387465;0.9505164;0.99532676;0.9995402;0.32377028;-0.31427425;-0.92977935;0.74828124 +0.77021635;-0.28380176;-0.63223803;0.98709947;0.9730316;0.99370223;0.43005443;0.98958206;-0.59130657;-0.9820163 +-0.96486443;0.990847;-0.997859;0.9998247;0.9791751;-0.0357527;0.22258474;0.97043407;-0.50227535;-0.31394258 +-0.08014693;0.99607354;-0.88523567;0.9992788;0.715602;-0.32893422;0.92566866;0.99483186;0.6354896;-0.9944459 +-0.28458008;-0.99627215;0.9938099;-0.98323435;0.92582643;0.99486357;0.8998966;-0.9721243;-0.0003347695;0.019740498 +-0.987834;0.99572116;-0.9991618;0.9997404;0.81406945;-0.80107254;0.381402;0.85670334;0.38856688;0.39041144 +-0.0459125;-0.9831463;0.46860313;0.84163857;0.98935276;0.99964905;0.8631555;-0.8194269;-0.86349475;0.5851337 +-0.38923487;-0.9960677;0.9943509;-0.9811556;0.9445166;0.9962563;0.8903113;-0.9786391;-0.13304774;0.24392186 +-0.22870189;0.99720776;-0.99210644;0.9999434;0.92253244;-0.6536077;0.25798187;0.9974562;-0.42338336;-0.937215 +0.892802;-0.9851848;0.12640212;0.8964324;0.9961643;0.9995803;-0.15404229;0.023313217;-0.96981084;0.70334303 +-0.99628186;0.9840373;-0.99951756;0.99903333;0.87901735;-0.68875945;0.6267663;0.07132698;0.6400348;0.89026785 +0.034652874;-0.99052215;0.36428407;0.7517988;0.99182093;0.9996321;0.4976765;-0.879137;-0.95866245;0.9111157 +-0.99579525;0.98672503;-0.99946064;0.99958754;0.91244245;-0.5537092;0.81477594;0.4099008;0.6209831;0.78590554 +-0.96917254;0.27161002;-0.9977454;0.99857116;0.9945775;0.97339636;0.73267233;0.8283606;-0.6336049;0.41872007 +0.9925303;0.9404861;0.72928435;0.99868304;0.017028235;0.12749203;0.24904683;0.9991185;-0.9497654;-0.99096143 +-0.9226741;-0.95668626;-0.1101279;0.2655604;0.9446857;0.99914163;0.7132443;-0.96055406;-0.41905767;0.8888746 +0.99573284;-0.53243697;0.9962477;-0.4378055;-0.9554775;-0.44265983;0.066451505;0.9585208;-0.2593385;-0.99860734 +0.97432166;-0.9528639;0.36414137;0.66011214;0.9868013;0.9988854;-0.14603782;0.876582;-0.793457;-0.8272512 +-0.044102862;-0.9854098;0.82865244;0.39838058;0.986237;0.9995035;0.8774686;-0.8735206;-0.5637728;0.14662415 +-0.8883889;-0.16368286;-0.9941898;0.99931943;0.99383456;0.9890197;0.9060477;0.891656;-0.42746595;-0.4395713 +0.9980333;0.7115808;0.98497957;0.89051116;-0.8024756;-0.60368043;-0.052033637;0.99468;-0.83061194;-0.99788845 +0.9887254;-0.8920296;0.99402;-0.9832575;-0.599268;0.85652024;-0.5258917;0.9612431;-0.016159186;-0.9963638 +-0.91880476;0.9637544;-0.99431926;0.99744445;0.98654354;0.5008282;-0.8036631;0.9299704;-0.9043226;0.68888056 +0.6591068;-0.9096088;0.8849062;-0.9778523;-0.4741031;0.86950314;-0.23246391;-0.3418988;0.98702794;-0.9874238 +0.3809242;0.9933751;-0.9727582;0.9999403;0.9420095;-0.30696788;0.115813315;0.99880373;-0.728735;-0.96804655 +-0.9884417;0.9966924;-0.99935323;0.99984175;0.61084336;-0.87072355;0.6889299;0.7747036;0.70415515;0.2917185 +0.98927164;-0.8903057;0.99717915;-0.93893963;-0.9186423;0.3583427;0.50839245;0.9316661;-0.06862126;-0.9975128 +0.9846929;0.83860236;0.6108861;0.99536055;0.65248656;0.77330106;-0.48359123;0.99866134;-0.9642436;-0.96844167 +-0.93681806;0.99097306;-0.9963974;0.99966323;0.9730432;-0.08356353;-0.38864192;0.97780144;-0.7247495;-0.107666485 +-0.90209955;-0.33188188;-0.9947445;0.99438155;0.997836;0.98980063;0.08153385;0.7751571;-0.90986913;0.77462286 +-0.9953422;0.94999564;-0.99925554;0.9935794;0.96268475;-0.07919077;0.017153231;-0.18411486;0.13616528;0.9684484 +-0.49201235;0.9801061;-0.9908866;0.99986666;0.9868473;0.44718063;0.06271469;0.9959933;-0.81739795;-0.8956987 +-0.9528743;0.8858884;-0.9960163;0.97993374;0.99146324;0.66642195;-0.89708257;0.573435;-0.89568;0.96515423 +0.9977044;-0.5299957;0.99844325;-0.448418;-0.8570172;-0.15661289;0.38193858;0.95385677;-0.7762921;-0.99552035 +0.053163078;-0.99379545;0.98446953;-0.9932208;0.5398369;0.9713433;0.8645922;-0.9428019;0.8710569;-0.6711479 +-0.96963215;0.9956528;-0.9980983;0.9992932;0.8278825;-0.77334076;-0.5505499;0.9306146;-0.14093214;0.4864647 +-0.9975403;0.97648937;-0.9996976;0.9953525;0.6404477;-0.8676332;0.09496604;-0.71270853;0.8608776;0.9216241 +-0.87469757;0.8820653;-0.99341816;0.99650526;0.9942767;0.8176185;-0.7369793;0.93253213;-0.9434789;0.69971216 +-0.6597977;0.99800473;-0.99439144;0.9999116;0.8887797;-0.6171363;0.7381063;0.99289936;0.20488915;-0.9196517 +-0.93212646;0.9925472;-0.99647206;0.9996883;0.9648629;-0.20753914;-0.43915015;0.98006755;-0.6879892;-0.16711743 +-0.197064;0.99516386;-0.99123335;0.999946;0.95373356;-0.49793643;-0.13401462;0.9976508;-0.7284236;-0.920594 +-0.98508364;0.9946211;-0.998859;0.99955934;0.87609595;-0.74265194;0.006407881;0.8803684;0.02826544;0.5642523 +0.95488757;-0.9744602;0.41492546;0.7620744;0.99555486;0.9994546;-0.36815155;0.5716491;-0.92306757;-0.049575653 +0.99554735;0.93497753;0.77118415;0.9979876;-0.30404058;-0.46982285;-0.43445945;0.9989631;-0.98170006;-0.98051447 +-0.98383206;0.9895306;-0.9986937;0.9962048;0.86197144;-0.7180816;-0.7876813;0.51187664;0.015863013;0.9353116 +0.954969;-0.94029814;0.9864922;-0.9897626;-0.7070241;0.8386918;-0.07366365;0.8596516;0.7232217;-0.9958704 +0.9964577;0.79345226;0.9009342;0.9812898;-0.11024878;0.66918594;-0.6754049;0.9973778;-0.9791261;-0.96607107 +0.94575876;0.95514834;-0.14905463;0.9991599;0.80185086;0.04625354;-0.79418385;0.9986532;-0.9764591;-0.93508726 +0.9856919;-0.5931684;0.80652636;0.71275026;0.846982;0.99485236;-0.40198064;0.99067646;-0.9087242;-0.9761733 +-0.99732137;0.95374197;-0.99957883;0.9983908;0.9456106;-0.18590008;0.70097095;0.0010023486;0.7346383;0.823852 +-0.28933635;-0.85983086;-0.96365553;0.98293144;0.99861676;0.9979686;-0.3759562;0.7708101;-0.9691967;0.7514918 +-0.9878856;0.97336704;-0.99858856;0.99935;0.98693293;0.2787984;0.06948238;0.85613424;-0.7033095;0.7025576 +-0.99625605;0.6741852;-0.9989774;0.9950159;0.9775714;0.8348712;0.64174354;0.08008245;0.37312785;0.8040098 +0.11595027;0.99436724;-0.9842993;0.99994624;0.9519245;-0.37212288;0.030456018;0.99839634;-0.73854715;-0.9517159 +-0.99750704;0.93459564;-0.99963564;0.99609196;0.9403984;-0.23321483;0.49165684;-0.32092664;0.74824226;0.9064248 +-0.980651;0.97952837;-0.9983756;0.9997507;0.9886339;0.33717012;0.35293615;0.93618715;-0.61249125;0.11900273 +0.7194617;-0.99421793;0.9831235;-0.9944263;0.84223205;0.98411673;0.4532706;-0.8459546;0.5508627;-0.67248064 +0.99885404;0.33382118;0.9928479;0.47269198;-0.83584994;0.17569233;-0.69924545;0.9886953;-0.4749465;-0.9971808 +-0.22286576;-0.9966142;0.99515045;-0.9826141;0.9480993;0.99612;0.90240467;-0.97419304;-0.26493385;0.22043215 +-0.97915536;0.99432087;-0.99874306;0.99983877;0.9601234;-0.39655495;0.44439492;0.9538631;-0.12563366;-0.2025932 +-0.9926891;0.99520755;-0.9995972;0.99916196;0.0853352;-0.9506813;0.19996013;0.20627913;0.8078727;0.65051794 +-0.994368;0.77978396;-0.9983687;0.99857056;0.96327245;0.789546;0.8805972;0.4678615;0.3597566;0.49814376 +-0.8594869;0.97700065;-0.99559116;0.99981964;0.990116;0.5168158;0.11726639;0.98875684;-0.7144445;-0.7464993 +-0.39741263;-0.3288086;-0.9866836;0.99607414;0.998769;0.99432;-0.14518458;0.9506734;-0.9574121;0.18539415 +0.93242276;-0.98747146;0.5267978;0.8344931;0.99563295;0.9995768;-0.1690274;-0.017315133;-0.97379833;0.6605033 +0.99466556;0.112073556;0.905444;0.8852548;0.36023426;0.9743857;-0.5101557;0.9945833;-0.95861006;-0.9722846 +-0.9603386;0.9975771;-0.99845344;0.9997344;0.77889293;-0.8401161;-0.22190866;0.9554834;0.2793621;-0.12425503 +-0.9950255;-0.530735;-0.94185066;0.6683291;0.69309175;0.9770167;0.77265304;-0.8484486;0.82312673;0.5202334 +0.9814371;0.68010384;0.50703996;0.99200386;0.69943446;0.9325916;0.3343539;0.99817055;-0.8619733;-0.98992836 +-0.94429237;0.99865377;-0.9988791;0.99990845;0.51339096;-0.915784;0.43756852;0.95916045;0.7802017;-0.679762 +-0.9949101;0.8238802;-0.9991243;0.98987794;0.9889347;0.79832596;0.04384788;0.08051544;-0.31008023;0.9497821 +0.9712969;0.9717252;-0.025404247;0.9996812;0.604117;-0.19701397;0.19707796;0.9993881;-0.9373155;-0.98387885 +-0.6832563;0.9985385;-0.99665725;0.99992496;0.5882876;-0.92272645;0.1861724;0.99157387;0.38145423;-0.8901437 +-0.028193353;0.99561995;-0.98561245;0.99993575;0.9465874;-0.16414788;0.63972574;0.99800074;-0.32242364;-0.972221 +-0.8861182;-0.936285;-0.86312944;0.8364713;0.9774558;0.9987285;0.62178946;-0.7919227;-0.6156456;0.83451706 +-0.99370646;0.994071;-0.9995723;0.99921423;0.37397268;-0.9261459;0.36142117;0.17601258;0.7318098;0.7707551 +0.5566229;0.9909007;-0.9545777;0.99993277;0.934177;-0.115976796;0.33794832;0.99907094;-0.7079879;-0.9783066 +-0.5271951;0.9980432;-0.9919916;0.9999134;0.8698885;-0.66518086;0.75742555;0.99478024;0.16109295;-0.93775165 +-0.96045107;0.4450511;-0.9961253;0.99960387;0.9840688;0.9522021;0.9432308;0.86535525;0.082044706;-0.4069609 +-0.9590803;0.7285206;-0.9971069;0.99689734;0.9953823;0.90828747;-0.2268107;0.8586828;-0.8859362;0.7572728 +0.9919952;0.66812295;0.7521554;0.98834014;0.34207377;0.90262425;0.22451328;0.99796414;-0.8978106;-0.98987526 +-0.99756896;0.96379584;-0.9996595;0.99362755;0.7888871;-0.76520324;0.16212961;-0.70248324;0.82475936;0.94235337 +-0.9668276;0.8220985;-0.9978355;0.99952906;0.9940672;0.8927688;0.6301527;0.9221;-0.6608664;0.080273695 +-0.8744268;0.97414446;-0.9927415;0.9990486;0.986792;0.45997474;-0.6813484;0.975299;-0.8962631;0.11401982 +-0.9828419;0.9871526;-0.99847674;0.9996646;0.982132;0.01692765;0.12473005;0.9257328;-0.64414316;0.42090976 +-0.12609647;-0.9816433;0.2841217;0.88280594;0.9896774;0.99963015;0.8825694;-0.785794;-0.8448032;0.5361944 +0.95785946;-0.9747881;0.98046595;-0.99194825;-0.19247384;0.91221607;0.124876685;0.69290286;0.60975045;-0.98936665 +0.9796197;-0.33454886;0.66672623;0.8726978;0.82856387;0.99218416;0.40013734;0.9952067;-0.7625516;-0.9917361 +-0.80279183;0.99331594;-0.99602854;0.9999103;0.9744924;-0.16601212;0.12895705;0.9930479;-0.5511546;-0.84524506 +-0.81561154;0.9847801;-0.995201;0.99985474;0.9866126;0.30899733;-0.04422832;0.99195457;-0.7250494;-0.78804654 +0.97294706;-0.76322067;0.98635185;-0.95240813;-0.9132937;0.42551965;-0.3538616;0.94044495;0.7281058;-0.9988358 +0.6118612;-0.98313385;0.9718091;-0.9936584;-0.08196439;0.9035494;0.76727724;-0.6467139;0.93224776;-0.94257385 +0.98598623;0.9517045;0.43601593;0.99872047;0.3810094;-0.28164202;-0.6411137;0.9989786;-0.9770455;-0.9657705 +0.9831719;-0.8964926;0.9959609;-0.9518112;-0.9245298;0.40356562;0.5235793;0.92455286;0.23010652;-0.9978091 +0.99567825;-0.45269912;0.9446958;0.560511;0.33213145;0.98895574;-0.54293436;0.99095523;-0.949692;-0.978308 +-0.99663115;0.7631925;-0.9949814;0.997842;0.8153519;0.60819095;0.9282879;0.06655435;0.8328545;-0.22114919 +-0.9110696;0.9986625;-0.99869245;0.99990094;0.43185443;-0.93707955;-0.04090591;0.9722331;0.6798466;-0.6962001 +0.31669098;-0.9851414;0.72431296;0.71117795;0.9903895;0.9996449;0.84948546;-0.79839045;-0.7915277;0.2708667 +-0.46756303;-0.99120975;0.18112852;-0.22589941;0.9869758;0.9993451;-0.14510806;-0.9722456;-0.95497763;0.9855994 +-0.9799502;0.9974478;-0.999292;0.9995771;-0.07983301;-0.9612661;-0.33836928;0.7906967;0.8084746;0.14182383 +-0.9973362;0.83767384;-0.99717504;0.9982869;0.8785489;0.5365479;0.9155584;0.108344406;0.8053845;0.07063572 +0.5207706;-0.55260783;-0.8673208;0.9914383;0.990758;0.9959076;0.5354109;0.9664881;-0.53673697;-0.9529085 +0.8636783;-0.98619676;0.016545841;0.8865899;0.9957524;0.999574;-0.25277176;-0.12016936;-0.9729008;0.7777935 +0.9048624;-0.102998324;-0.1533003;0.9791769;0.97027385;0.9926266;-0.3304921;0.9949084;-0.91125023;-0.96095586 +0.9773283;0.9468969;0.28656963;0.9992089;0.6147979;0.1759661;-0.22018598;0.9991896;-0.9591761;-0.97384316 +0.9161715;-0.7397881;-0.11717037;0.9407459;0.98272157;0.9975671;-0.27184072;0.9767524;-0.8343229;-0.93641657 +0.92445654;-0.9805446;0.98720723;-0.9924082;-0.26603046;0.8974763;0.6829033;0.45034307;0.6451939;-0.9845634 +-0.9894269;0.99575406;-0.99949384;0.9990779;0.27863783;-0.942824;-0.22501169;0.4833005;0.7754475;0.64437413 +-0.89179707;0.9912075;-0.9942108;0.9983065;0.92212045;-0.323768;-0.90381527;0.96167463;-0.80491203;0.45286092 +0.98348284;0.77531004;0.9480323;0.8041413;-0.9789726;-0.9553188;-0.05589895;0.9939817;-0.007956069;-0.99972326 +0.99666214;0.081944026;0.93692046;0.85215545;0.0731195;0.96879655;-0.5080855;0.99399203;-0.9637036;-0.9741003 +-0.9482876;-0.3926747;-0.9953714;0.9965705;0.9948384;0.9908071;0.74936336;0.66949445;-0.62790287;0.512129 +-0.9969059;0.9752921;-0.99959016;0.9973083;0.8571876;-0.7069981;0.41563973;-0.3389663;0.71524906;0.9296695 +-0.5679622;0.9985197;-0.99535376;0.999923;0.57663566;-0.92465425;0.24468234;0.9939217;0.2941947;-0.91483694 +-0.81459206;0.41640472;-0.99321187;0.99251634;0.99760175;0.9661698;-0.6025284;0.8985664;-0.9513951;0.7724475 +-0.84729064;-0.98354757;0.65268904;-0.37261978;0.96608126;0.99946666;0.70382583;-0.97742975;-0.7158288;0.9394178 +-0.98959416;0.9635381;-0.9987105;0.9961704;0.98602927;0.24337433;-0.42950904;0.53518593;-0.6514243;0.9429638 +-0.99483615;0.9370083;-0.99911416;0.9987286;0.98579466;0.59423697;0.44685578;0.6626804;-0.44158837;0.79525 +0.84686136;-0.9840588;0.9653133;-0.99408144;0.42939436;0.938456;0.31921455;-0.29980156;0.8127558;-0.95122015 +0.99917954;0.26483;0.9956551;0.43542388;-0.72450227;0.19598487;-0.47839886;0.98516804;-0.8322289;-0.9938633 +-0.9864463;0.9961851;-0.9991845;0.99970794;0.7147945;-0.8595923;0.26935697;0.8408732;0.5187636;0.40521568 +-0.97723883;0.99759823;-0.99921525;0.9997376;0.27856538;-0.93490976;-0.0807945;0.8687269;0.7496959;0.033098232 +-0.9874929;0.98123777;-0.998816;0.9947953;0.95790666;-0.3325078;-0.6870909;0.46183082;-0.3140199;0.954308 +-0.88128924;0.9522209;-0.9931606;0.9987533;0.9917246;0.67078876;-0.61199754;0.9683428;-0.9163544;0.2564045 +-0.90016973;0.986448;-0.9957926;0.99980634;0.9842711;0.20820805;-0.1318031;0.9866328;-0.72814345;-0.55695915 +-0.89158577;0.9917972;-0.99693054;0.99989325;0.9796148;0.022535138;0.3092815;0.98861974;-0.44301537;-0.8059467 +-0.9965654;0.9788137;-0.9995469;0.9981367;0.8797492;-0.6756027;0.4855888;-0.14374709;0.65469426;0.9207257 +-0.9964791;0.95522106;-0.9993072;0.9983529;0.97723126;0.35206774;0.6015759;0.4277066;0.14112215;0.83156264 +-0.983089;-0.46261245;-0.9753489;0.9845777;0.9221467;0.9892609;0.9038735;-0.31034213;0.29699907;0.13598575 +0.95864964;0.99131674;-0.6528456;0.9996872;-0.2425343;-0.960292;0.5404868;0.9993264;-0.75695187;-0.9952342 +-0.9759361;-0.75610334;-0.93502754;0.91583884;0.9266586;0.99599075;0.8282357;-0.734271;0.13156633;0.54566216 +-0.91288286;0.94786;-0.9942669;0.9970517;0.9898199;0.62123257;-0.7916318;0.92470014;-0.9177632;0.7176254 +-0.8153907;0.24062237;-0.9951211;0.99894017;0.9977931;0.9832797;0.47746748;0.9472486;-0.86275923;-0.10523301 +0.8009342;-0.9902756;0.22615822;0.76851445;0.993124;0.99953353;-0.4880056;-0.56308;-0.98209673;0.9030469 +0.9858918;-0.5736777;0.7343701;0.7091357;0.82079124;0.9935365;0.40672183;0.9934546;-0.755182;-0.9913831 +-0.9853168;0.99746823;-0.9993766;0.9998345;0.37605518;-0.9212932;0.46466422;0.8164974;0.7927181;0.05783666 +0.302607;-0.9889318;0.95914024;-0.99111843;0.6214121;0.9769989;0.26061556;-0.9121328;0.9295044;-0.78681767 +-0.59770644;0.9979359;-0.9932272;0.9999118;0.89040554;-0.5914093;0.756789;0.99395734;0.1744079;-0.9320143 +-0.86122555;0.99546283;-0.99687517;0.9998975;0.9555736;-0.48109758;-0.13415231;0.99112594;-0.43958938;-0.75594366 +0.96372056;0.8567008;0.9137537;0.7522985;-0.9833805;-0.9675771;-0.13494766;0.99364626;0.4583334;-0.999776 +0.99551904;0.50683993;0.8488839;0.9739897;0.049504563;0.9204066;0.28420997;0.99725443;-0.88489616;-0.99230963 +-0.9681006;0.0062306016;-0.9972569;0.9973553;0.9948454;0.98195505;0.6906628;0.73987705;-0.65211904;0.5862753 +-0.9876046;0.9972056;-0.9994026;0.9998198;0.33990082;-0.9246863;0.5247329;0.77321863;0.79440033;0.15177214 +-0.6179919;0.9820859;-0.9926576;0.99986726;0.9869676;0.40258005;0.05470141;0.9950245;-0.7926855;-0.8759409 +0.96352667;-0.9823034;0.6318885;0.5074572;0.993334;0.9994697;0.2370538;0.42933938;-0.85382825;-0.42423466 +-0.9461595;0.9879352;-0.99759924;0.99987453;0.9839739;0.20046425;0.43071502;0.97881526;-0.39671582;-0.67643785 +-0.7108403;0.9975812;-0.9971617;0.9999247;0.8887756;-0.7967446;0.0026881965;0.9932741;-0.15294223;-0.8412413 +-0.5298587;0.46753138;-0.99022156;0.997941;0.9981603;0.9797262;-0.15415095;0.9769632;-0.9473218;-0.16472916 +-0.9846707;0.98881984;-0.9987009;0.9979334;0.9446715;-0.4882458;-0.5512215;0.7542648;-0.3551724;0.88172823 +-0.99614716;0.9540439;-0.99929106;0.9971013;0.97396874;0.16177425;0.38160095;0.17887951;0.10614212;0.922069 +-0.8657972;0.99502236;-0.99555516;0.9997693;0.9287266;-0.49930608;-0.6705862;0.98910993;-0.6447534;-0.49713707 +-0.39424625;-0.7152848;-0.97772855;0.9899205;0.99879164;0.9964241;-0.3864973;0.87054616;-0.97188807;0.69801027 +-0.7831414;0.9828994;-0.9911313;0.9995443;0.97910446;0.24571988;-0.6848492;0.9895982;-0.87895185;-0.40803087 +-0.98614013;0.99645925;-0.99935365;0.99935454;0.298702;-0.94072795;-0.18682173;0.67430174;0.736967;0.54515976 +0.8853203;-0.994536;0.9899235;-0.9795428;0.98415625;0.99764514;0.23388034;-0.7928687;-0.58501786;0.1693584 +-0.909243;0.9985037;-0.9986022;0.99990857;0.62189746;-0.9072146;0.16739771;0.97412676;0.61801046;-0.69223243 +0.905516;-0.98993593;0.987406;-0.9940331;0.43618232;0.94347566;0.8124882;-0.058369815;0.3412263;-0.9493975 +-0.861131;-0.9111633;-0.8998631;0.9363396;0.97887975;0.9986288;0.77326286;-0.6211553;-0.5458618;0.64759994 +-0.9680354;0.9954825;-0.99801165;0.9995735;0.8972837;-0.6654975;-0.37307116;0.95269823;-0.31010514;0.2929593 +0.76228446;-0.9870837;0.10144505;0.9344746;0.99574494;0.9996218;0.48162106;-0.2666155;-0.95084506;0.64218754 +-0.9910176;0.99599785;-0.99953085;0.99962264;0.2995333;-0.9327242;0.3883121;0.5667205;0.7727509;0.5053761 +0.9445341;-0.98264056;0.4471054;0.8230387;0.99617666;0.9995602;-0.17915216;0.28226683;-0.95504135;0.3936432 +0.35044622;-0.99020517;0.61113316;0.8103863;0.9918646;0.99966514;0.62686723;-0.83417314;-0.95514256;0.8146096 +0.9940162;0.9184785;0.8075901;0.99670297;-0.032147996;0.07695673;-0.5269752;0.9986208;-0.9782117;-0.9740295 +0.65494514;-0.9934346;0.96791106;-0.9594666;0.9419395;0.99623966;-0.2142377;-0.8389821;0.38119662;-0.37135532 +-0.992717;0.9800179;-0.9990365;0.998037;0.969125;-0.21427974;-0.12993872;0.5087785;-0.3221051;0.92512554 +-0.9782406;0.98038435;-0.99834913;0.99981236;0.98847824;0.36134958;0.4773036;0.94488436;-0.5337534;-0.11576921 +-0.93313575;0.9256915;-0.99558866;0.99568486;0.99187005;0.6774773;-0.77977586;0.8775441;-0.9121662;0.8318644 +-0.9950118;0.7489264;-0.99835736;0.9981799;0.96246433;0.79730356;0.8633263;0.37852523;0.4089102;0.54105794 +-0.8933561;-0.981803;0.5027042;-0.54733056;0.96182024;0.9993157;0.6069351;-0.9845527;-0.698467;0.96740997 +-0.988853;0.7257314;-0.99682105;0.9993898;0.94276047;0.8109807;0.94663715;0.65277076;0.4842745;-0.11258816 +-0.9879622;0.9906688;-0.99907136;0.99985135;0.97336644;-0.13542242;0.62343496;0.9190915;-0.19522756;0.02277895 +0.98479456;-0.85287327;0.99317706;-0.960762;-0.87189263;0.5820516;-0.16068499;0.9511181;0.30169263;-0.99814296 +0.9880789;-0.9216776;0.9940889;-0.98802155;-0.45595232;0.9000368;-0.46882573;0.95622456;-0.051425174;-0.9949325 +-0.97843117;0.99580467;-0.998716;0.9997623;0.9155013;-0.66051245;0.1729167;0.9446482;-0.0062622055;0.052132677 +0.99132705;0.5979355;0.7368236;0.98504436;0.41085738;0.92822814;0.28583533;0.99778074;-0.87925893;-0.9908264 +-0.97303843;0.99794006;-0.9992109;0.9997734;0.2532357;-0.940581;-0.10166238;0.8916989;0.76923335;-0.13527672 +-0.96169883;-0.7978272;-0.97560424;0.94856274;0.9803165;0.99616146;0.59448373;-0.502118;-0.45835006;0.82747984 +0.9968069;-0.5160568;0.99284256;-0.7801557;-0.88678586;0.39077944;-0.7381681;0.97547317;-0.13245672;-0.99847734 +-0.9287471;0.9834516;-0.996241;0.99975073;0.98771375;0.32186082;-0.08263534;0.9803526;-0.74140173;-0.38091195 +-0.9844335;0.7280954;-0.99834514;0.9982173;0.9925896;0.90973026;0.36843094;0.8073089;-0.745581;0.69074774 +-0.98667765;0.9938315;-0.99889785;0.9996889;0.9298539;-0.59959126;0.2130258;0.89727277;-0.17633033;0.48535487 +0.35267055;-0.9848046;-0.61760753;0.94391423;0.9946862;0.9995045;0.24110359;-0.4053877;-0.93127275;0.79847753 +0.21898408;-0.9906524;-0.25527933;0.7972404;0.9923705;0.9994947;-0.13867858;-0.7705862;-0.96461207;0.93954796 +0.99750143;0.46819735;0.93635863;0.93730676;-0.21400478;0.9032525;-0.5211719;0.99556637;-0.97131705;-0.97369546 +0.7388795;-0.99291396;0.78906935;0.55209965;0.9911789;0.99960715;-0.02861339;-0.84386545;-0.9807341;0.8939648 +0.6034046;-0.5284046;-0.8285017;0.98968107;0.98780733;0.99569565;0.49410042;0.9732767;-0.53168464;-0.9638535 +-0.99790484;0.9335538;-0.9995236;0.9981141;0.9389041;-0.04936325;0.7218937;-0.107915916;0.8087674;0.73486555 +-0.9656487;0.9655652;-0.9972873;0.9994796;0.9919869;0.5498459;-0.027141873;0.9452862;-0.8021527;0.2850576 +-0.9932036;0.99458545;-0.9995729;0.99927086;0.29566562;-0.9331713;0.37137786;0.23239513;0.7518466;0.72767717 +-0.9330019;0.99807763;-0.9982534;0.9997835;0.6880229;-0.8862927;-0.3895787;0.9679983;0.3079779;-0.35117874 +0.7604985;0.98135066;-0.8399182;0.9996423;0.891706;-0.25009224;-0.8984854;0.9978002;-0.97325945;-0.8246376 +-0.6516308;0.6336277;-0.99152553;0.9984969;0.9978259;0.969591;-0.07807857;0.97803193;-0.9361228;-0.21994887 +-0.9914638;0.98984766;-0.99910766;0.99970776;0.9645998;-0.30317718;0.4226579;0.8475774;-0.3262111;0.60213494 +0.84842944;-0.9783767;-0.06087361;0.9288745;0.996723;0.99953413;0.51618195;0.2055969;-0.8709676;0.06182751 +-0.99313486;0.97376275;-0.9990874;0.9961722;0.9676169;-0.1776803;-0.2786017;0.26818037;-0.24073909;0.9566567 +0.823656;0.9844191;-0.81327474;0.99983066;0.8766936;-0.32347614;-0.5671238;0.9989709;-0.9373126;-0.94804907 +0.94140035;-0.98549306;0.58646494;0.7381169;0.995235;0.9995411;0.41075984;0.172153;-0.89746207;-0.06961033 +-0.798371;-0.90621626;-0.964229;0.9781173;0.99206847;0.99831015;0.60843873;-0.10577794;-0.7499545;0.7000535 +-0.80844617;0.99778175;-0.99666584;0.9998528;0.7904298;-0.85270673;-0.63747907;0.9907436;-0.23242761;-0.68625265 +0.37308475;-0.99195963;0.93680644;-0.020187972;0.98612404;0.9996483;0.5952074;-0.9445037;-0.93464345;0.8170323 +-0.9505582;0.9896962;-0.9976631;0.99987304;0.98175687;0.08972438;0.37592956;0.9781066;-0.40720463;-0.6193689 +-0.6045359;-0.020348424;-0.9896877;0.9943849;0.9986393;0.9881394;-0.42992795;0.92918724;-0.963817;0.5904195 +0.99740434;-0.25186446;0.9970797;0.030561656;-0.907788;-0.4921403;0.25212228;0.9630815;-0.7028576;-0.99726385 +-0.9888234;0.9610681;-0.9985286;0.99886155;0.9882751;0.40806615;-0.044710588;0.7915263;-0.7342115;0.8150523 +-0.24449179;0.9878385;-0.9855324;0.99990565;0.9747581;0.05187196;-0.22658846;0.9972518;-0.85948575;-0.9001037 +-0.040813785;0.98842496;-0.9699661;0.99975955;0.9305158;-0.2774903;-0.89959246;0.995785;-0.9358007;-0.73036766 +0.4003658;-0.98957676;0.5659864;0.8533339;0.992405;0.99966323;0.6877777;-0.7860436;-0.9493398;0.75388193 +-0.777222;0.8636341;-0.9923209;0.99899256;0.9963258;0.90817034;-0.25288388;0.98059225;-0.9286786;-0.1883018 +-0.58430487;-0.9897321;0.5350423;-0.07937408;0.9855147;0.9995623;0.38314867;-0.9727707;-0.91993254;0.9675988 +-0.10478168;0.9788026;-0.97409505;0.9997471;0.97315884;0.24576928;-0.7362119;0.9960425;-0.9370417;-0.79494876 +-0.9644694;0.7240619;-0.9972842;0.99639726;0.9950877;0.90529454;-0.25783598;0.83331805;-0.8831244;0.8004243 +0.65046346;0.9796691;-0.89241695;0.99974644;0.92774206;-0.029574687;-0.8394051;0.9978653;-0.9660122;-0.8481158 +-0.96029603;0.9211356;-0.9970319;0.9990896;0.99417484;0.75606745;-0.058410577;0.9297275;-0.84927404;0.45832807 +0.9199533;0.97968674;-0.64372444;0.99984914;0.81992775;-0.0813846;0.40861547;0.9994759;-0.8658901;-0.9839839 +-0.602778;-0.79693425;-0.9775413;0.9746907;0.99848205;0.9959984;-0.5598724;0.6485474;-0.96988475;0.90548015 +-0.9760082;-0.6695082;-0.9871231;0.96950924;0.98019916;0.9932931;0.59396464;-0.3397528;-0.35421264;0.81858355 +-0.99127233;0.9934367;-0.9994513;0.9989367;0.6784657;-0.87697184;-0.08607521;0.44910365;0.62366647;0.8106412 +-0.93696;0.9953997;-0.9966656;0.99928;0.8684959;-0.64720106;-0.7718957;0.9650454;-0.5458879;0.2659005 +-0.95959574;0.99800307;-0.9989516;0.9998724;0.7758078;-0.8500718;0.32062742;0.95047766;0.5660652;-0.40295714 +0.9908527;0.7258819;0.9686974;0.85642046;-0.9653192;-0.9350114;0.15058964;0.9936609;-0.46210778;-0.9995049 +-0.9946984;0.9564522;-0.9990819;0.9988844;0.98420995;0.43841597;0.37444824;0.6631386;-0.46615544;0.8162415 +-0.8891996;0.9346135;-0.9937037;0.9985683;0.9933359;0.7426526;-0.5657946;0.9620797;-0.9220017;0.35357127 +-0.9939668;0.99060744;-0.9992968;0.99970365;0.94788706;-0.4202939;0.6144369;0.74982613;0.014872185;0.66322196 +-0.98869234;0.9955751;-0.99920195;0.99978185;0.8541224;-0.7547656;0.49680704;0.8654809;0.35579872;0.32031173 +0.1461844;-0.99332196;0.9772382;-0.98384684;0.8901727;0.99382085;0.2606505;-0.9508255;0.71589804;-0.48904595 +-0.98492754;0.9836644;-0.9985696;0.99965936;0.9849296;0.13253817;0.19573401;0.91453874;-0.6450412;0.445519 +0.021993367;-0.99099755;0.4053164;0.6960487;0.9916578;0.99962723;0.43182003;-0.8966461;-0.9611585;0.92577034 +-0.09153868;-0.9732056;-0.7688738;0.98115647;0.9934017;0.999416;0.85525894;-0.28359154;-0.6936197;0.23431939 +-0.99446994;0.98680913;-0.9993149;0.99896795;0.9175344;-0.596836;0.25217062;0.38306403;0.23882185;0.8966694 +-0.92824894;0.95128214;-0.9949607;0.9957999;0.98800755;0.5570327;-0.83534604;0.88985413;-0.90500456;0.8145575 +0.8520698;-0.98451805;0.2457457;0.9211592;0.99645597;0.9995767;0.5510245;-0.03789704;-0.925045;0.32765704 +-0.99325854;0.9824291;-0.9991611;0.99967355;0.9770495;0.027867302;0.53288037;0.8204992;-0.3705351;0.60292983 +-0.98736936;0.9695947;-0.9985351;0.9993053;0.9879386;0.34297174;0.054406233;0.8566406;-0.71958464;0.7024874 +-0.9955947;0.9823104;-0.99928534;0.9992628;0.9594188;-0.24638182;0.5938113;0.5292997;0.11201862;0.840136 +-0.9391986;0.13791609;-0.9965399;0.9957399;0.9972729;0.97852635;-0.015917562;0.80776775;-0.8926835;0.76421916 +-0.9961438;0.46550083;-0.98868686;0.9899356;0.7526185;0.8605455;0.9074594;-0.20687258;0.8191266;-0.08368979 +-0.99579376;0.9795836;-0.9993304;0.99945617;0.9702608;0.041099012;0.7004424;0.68591404;0.031687256;0.692058 +-0.92797095;-0.96046203;-0.1045254;0.10347103;0.946312;0.999094;0.66071314;-0.96827805;-0.46162987;0.9241572 +0.4475164;-0.9885843;0.96053404;-0.99255055;0.58037204;0.9707935;0.2922887;-0.8761232;0.9232027;-0.81551784 +0.44134715;-0.99507684;0.9864862;-0.98948157;0.926205;0.99460095;0.38832667;-0.9414177;0.42699698;-0.34187922 +0.05799807;-0.9866809;0.7544794;0.6482357;0.9879239;0.99967366;0.7794553;-0.89490044;-0.90451497;0.7411064 +-0.94030833;0.9904217;-0.9965181;0.9996812;0.97598463;-0.03865915;-0.32342768;0.97716373;-0.72174066;-0.12591648 +-0.9216412;0.996916;-0.99658525;0.999201;0.6956002;-0.82738066;-0.8593427;0.9625923;-0.28315103;0.08800291 +-0.92441064;0.97286326;-0.99456555;0.9970745;0.98068583;0.35670662;-0.8466267;0.92187554;-0.88543856;0.72586775 +-0.9938281;0.9786482;-0.9991913;0.99964964;0.9785178;0.13256098;0.5816652;0.80845124;-0.33750987;0.59030694 +0.835354;-0.8379555;-0.6474829;0.9580522;0.9849527;0.9978251;0.5130363;0.9092629;-0.3587883;-0.9374819 +0.4802787;0.9560501;-0.9338226;0.9997229;0.9744438;0.59045;-0.537695;0.99790573;-0.9469457;-0.8948905 +-0.6855819;0.9858338;-0.99342716;0.9998668;0.9842171;0.23839307;-0.14259154;0.99453866;-0.7824813;-0.8395896 +-0.8312256;0.8549107;-0.99225754;0.9972548;0.99546754;0.8706429;-0.65117764;0.9539515;-0.94947237;0.5305877 +-0.7676673;0.9720023;-0.9915034;0.99959075;0.98870206;0.5283654;-0.5030047;0.99008685;-0.88844794;-0.49723503 +-0.99755013;0.97718376;-0.9996991;0.9956639;0.63267136;-0.87247753;0.12235813;-0.7070934;0.86218137;0.9187826 +-0.9631365;-0.9768328;0.96897393;-0.98360115;0.4427075;0.9872935;0.72570354;-0.98654246;0.92116857;0.04052174 +-0.64131075;-0.9933135;0.9879433;-0.97934645;0.8459546;0.9917302;0.87030935;-0.97455996;0.59496695;-0.3096044 +0.998666;0.7440753;0.9835548;0.89016086;-0.8221649;0.21689473;-0.6327688;0.9964896;-0.29839697;-0.99771696 +0.99789274;-0.30488703;0.99593437;-0.306387;-0.9065884;-0.15315555;-0.44920814;0.9667338;-0.5545661;-0.99787664 +-0.9858241;0.98046577;-0.998643;0.9996988;0.98662084;0.23479371;0.30944625;0.9125001;-0.61577594;0.37543657 +0.9673231;-0.96163464;0.27861598;0.69660807;0.99039036;0.9990531;-0.047548965;0.8260918;-0.77038145;-0.7606177 +-0.84615517;-0.8754642;-0.9743068;0.9818432;0.9924172;0.9977754;0.59088254;0.019473273;-0.7630894;0.7257467 +0.99427223;-0.7465496;0.9970978;-0.7675712;-0.9480714;-0.15393376;0.25861132;0.94775885;-0.20501918;-0.9982529 +0.4789521;-0.9945789;0.9887009;-0.9705691;0.96997297;0.99753624;0.4524271;-0.9373845;-0.017101796;-0.00070097286 +-0.20247187;0.9980088;-0.99323535;0.9999321;0.68206775;-0.91119254;-0.28006202;0.99656504;-0.30988348;-0.90962213 +-0.97923344;-0.6712433;-0.95606405;0.95282394;0.92596555;0.9944868;0.8584261;-0.6138696;0.193043;0.42370477 +0.1291624;-0.9765084;-0.8335521;0.8892758;0.99680835;0.9991844;-0.6147896;-0.16447598;-0.9633311;0.9174586 +-0.8593829;0.9306849;-0.9928341;0.9989211;0.9941484;0.7826134;-0.47497812;0.97303784;-0.9245863;0.11825129 +0.9084658;-0.486706;-0.13606983;0.9633153;0.974957;0.99597734;-0.30904827;0.990659;-0.8693455;-0.9600276 +0.9904882;0.97744524;0.354564;0.999415;0.15355414;-0.35205984;0.7574223;0.9995367;-0.7861243;-0.9979433 +0.9533115;-0.9010736;0.98694104;-0.98792064;-0.8428782;0.73421973;-0.17502163;0.890759;0.80508703;-0.9973299 +0.2572998;0.99785644;-0.9763538;0.9999228;0.6028413;-0.91093946;0.5700053;0.9983618;-0.2785683;-0.97491133 +0.90144336;-0.9939775;0.977888;-0.9586772;0.9765004;0.9969855;-0.30857697;-0.5703172;-0.4154807;-0.12872739 +-0.98332626;0.98969686;-0.9985205;0.9995516;0.9726332;-0.19954509;-0.040782724;0.9122467;-0.61603343;0.5838097 +0.7899539;-0.9912939;0.7005723;0.7952399;0.9930618;0.99962944;0.27381027;-0.66866463;-0.975947;0.78415126 +-0.939079;0.76751536;-0.9956469;0.98080224;0.9946399;0.8441875;-0.84768504;0.66995746;-0.92516804;0.95254993 +-0.9949133;0.98790735;-0.999344;0.99967444;0.9589516;-0.25507337;0.6968193;0.7374829;0.05276631;0.6427641 +-0.8398726;0.9893117;-0.99238855;0.99933743;0.9575509;-0.09076614;-0.8220597;0.983924;-0.8463452;-0.12182112 +0.8985158;-0.9474174;-0.2703507;0.90614516;0.9950842;0.99911803;0.088077106;0.7607139;-0.7460711;-0.64516884 +0.9367917;-0.8404158;0.9888573;-0.9641724;-0.9609555;0.28402352;0.44022366;0.86180776;0.8678187;-0.99850214 +0.05983121;-0.99231905;0.9501513;-0.38169578;0.9872326;0.9995782;0.82077247;-0.9474795;-0.82337356;0.71997535 +0.5909392;0.9726568;-0.9092731;0.9997741;0.953882;0.25679797;-0.71783483;0.99799293;-0.9598126;-0.875661 +-0.9418464;0.99466676;-0.99677265;0.9992845;0.9035346;-0.5618692;-0.7296763;0.96397585;-0.6021138;0.32893035 +-0.99766827;0.90994644;-0.99964654;0.9876094;0.91056234;-0.3723357;0.17512052;-0.68119097;0.7973381;0.9510743 +-0.99617136;0.9804064;-0.99946773;0.99945104;0.93595254;-0.32839546;0.84261477;0.4177257;0.6453971;0.72908384 +-0.99621433;0.98721236;-0.9995594;0.9988474;0.81356716;-0.7862744;0.5072161;-0.03248616;0.65925455;0.89536667 +-0.9958342;0.9391898;-0.9991605;0.99898505;0.9713258;0.5373784;0.81451875;0.5874301;0.3204559;0.59796166 +0.91902024;0.9878468;-0.65920347;0.9998572;0.75728583;-0.23431373;0.75222117;0.99949515;-0.69804996;-0.99459666 +-0.98457074;0.9943974;-0.99881655;0.9997164;0.9277613;-0.610892;0.19996822;0.9176725;-0.16407754;0.37260315 +-0.92627525;0.98006326;-0.99472684;0.9971458;0.9728006;0.20226635;-0.8607079;0.9264096;-0.8634679;0.7082105 +-0.27017492;-0.99615204;0.99291193;-0.9834415;0.91381395;0.9941526;0.8929777;-0.96997786;0.13032055;-0.08623984 +-0.95797056;0.4501779;-0.99746984;0.99927825;0.9949685;0.9672898;0.7871353;0.8970033;-0.6139162;0.0927317 +0.9250086;0.8638963;-0.22837521;0.9986276;0.92777926;0.8088474;-0.35146394;0.9988135;-0.95445895;-0.95893437 +-0.67170495;0.9982414;-0.99534094;0.9999212;0.8418455;-0.7803562;0.6237258;0.9926646;0.22237763;-0.8990789 +0.9295582;-0.6740597;-0.15093775;0.93227834;0.9557687;0.9961753;0.38174605;0.98025084;-0.54499406;-0.9806048 +-0.8246744;-0.98896766;0.98403305;-0.9724659;0.7860006;0.9907538;0.8404223;-0.9748876;0.77442014;-0.35128498 +-0.64227283;-0.99505174;0.99238855;-0.98835003;0.92172897;0.9961367;0.8406202;-0.98655546;0.14315437;0.39148226 +-0.9834907;0.99091804;-0.99887556;0.9998692;0.97221863;-0.095259346;0.6664632;0.9398116;-0.08395282;-0.31214574 +0.67639476;-0.9924275;0.6376365;0.7219465;0.9918036;0.9996181;0.055307128;-0.80008024;-0.97946805;0.8947162 +-0.52421606;-0.22239074;-0.98822826;0.9950213;0.99880725;0.9918551;-0.3346604;0.93280345;-0.9654299;0.51454735 +-0.7391382;0.97356814;-0.99411863;0.99982893;0.9902222;0.59402;0.19898379;0.99265623;-0.7591976;-0.8486874 +-0.19125758;-0.98743135;0.8667972;0.23853874;0.9863857;0.9995591;0.86971396;-0.9143172;-0.6980067;0.47104794 +-0.9882656;0.35196054;-0.9980065;0.9968068;0.98668396;0.9514754;0.77660894;0.4776987;-0.2069355;0.7090821 +-0.9947467;0.9800756;-0.99942803;0.9952355;0.81295294;-0.73740715;-0.33126476;-0.35096112;0.59962684;0.9566241 +-0.9888403;0.992104;-0.998974;0.99968994;0.95351857;-0.4480305;0.28008872;0.8795719;-0.31426588;0.5602568 +0.9517643;-0.97377497;0.99322766;-0.9903136;-0.6377439;0.8474663;0.6671917;0.755745;0.5532355;-0.99175316 +0.9983121;0.5424902;0.9530094;0.9309836;-0.445598;0.8448813;-0.59469134;0.995005;-0.979473;-0.9700408 +-0.9984535;0.6588924;-0.9988447;0.9820448;0.92001706;0.5630953;0.5416557;-0.493099;0.88560086;0.71424425 +-0.95671844;0.2664132;-0.9967709;0.9933517;0.99652654;0.9708159;-0.19057262;0.74145746;-0.8970945;0.86127377 +0.7895316;-0.9972671;0.9966298;-0.9958503;0.83178777;0.98366433;0.9180033;-0.7920377;-0.39410266;-0.5246105 +0.99929875;0.4953539;0.99510825;0.70958674;-0.676712;0.1208445;-0.42240345;0.99008197;-0.90412664;-0.9918879 +0.9954401;0.7336203;0.9661179;0.7262125;-0.9342078;-0.7069533;-0.67220265;0.9950529;-0.096738204;-0.9994804 +0.92175126;-0.9934031;0.9768695;-0.96354693;0.9753902;0.99645036;-0.31581494;-0.43445686;-0.41113192;-0.22729862 +0.8824897;-0.9907633;0.60337055;0.7059027;0.99301416;0.9995662;-0.38773566;-0.52558386;-0.9845687;0.85789585 +0.99823415;0.560082;0.9905828;0.7569642;-0.8242441;-0.5083561;-0.13773094;0.9905275;-0.8164508;-0.99752223 +-0.060324002;-0.9680559;-0.9196892;0.97866815;0.99637496;0.9992415;0.31125435;0.09228182;-0.8772174;0.6292523 +0.9063583;-0.9815781;0.04969145;0.89941055;0.996555;0.9995683;-0.16192923;0.23460868;-0.9579446;0.5549055 +-0.9948053;0.98190725;-0.9992178;0.9985377;0.9466155;-0.41370484;0.21170442;0.33315498;0.060916144;0.92284155 +-0.9067761;0.998356;-0.9981405;0.99982655;0.6391767;-0.90943205;-0.437839;0.97537947;0.33105662;-0.507994 +0.84353125;-0.89964336;0.9369516;-0.98198855;-0.72451854;0.80393964;-0.26408213;0.3689621;0.9681389;-0.9952893 +-0.9931694;-0.86953425;0.7407698;-0.9168951;-0.6328476;0.9583956;0.19831084;-0.9790849;0.9835427;-0.20603709 +-0.99655926;0.9742732;-0.99951833;0.9991714;0.9375736;-0.34840494;0.7831402;0.25714362;0.665345;0.8071815 +-0.38160312;-0.79667634;-0.9732099;0.9870841;0.99870944;0.9971932;-0.38344663;0.82647514;-0.9696167;0.7346521 +0.9963708;0.95108026;0.7658278;0.9987997;-0.4676079;-0.66975486;0.27460724;0.9991691;-0.9663839;-0.9931078 +-0.98465;0.039713044;-0.9978567;0.97307956;0.99376774;0.96795106;-0.101602666;-0.004763571;-0.6685102;0.96463543 +-0.9887383;0.9900546;-0.99908066;0.9974449;0.85827225;-0.75180477;-0.52613884;0.4675377;0.22974545;0.9211276 +-0.997014;0.9818856;-0.999643;0.99699414;0.7136591;-0.8411446;0.22471394;-0.5178394;0.78776544;0.9200049 +0.9876786;0.123259366;0.8074113;0.93064225;0.70184773;0.9820087;-0.5399476;0.99446034;-0.95493853;-0.9663826 +-0.9508172;0.94228727;-0.99659383;0.99933493;0.99411815;0.71978;-0.052292123;0.9503351;-0.8452372;0.25266832 +-0.43444553;-0.99223495;0.42628887;-0.46958518;0.9859734;0.99939156;-0.16715898;-0.9813544;-0.9569528;0.9886183 +0.787912;0.96161246;-0.7729257;0.9997016;0.93467325;0.34513402;-0.650293;0.9985453;-0.962454;-0.9161988 +-0.9506124;0.99466735;-0.99706525;0.99858934;0.83337575;-0.7051192;-0.8404787;0.93122154;-0.4838559;0.5972012 +-0.77102065;0.9875991;-0.9907817;0.9995381;0.9648162;0.0052825892;-0.788837;0.9897138;-0.8668456;-0.3940885 +-0.7653315;0.46822655;-0.9919286;0.9939435;0.9977431;0.965802;-0.5824585;0.924112;-0.95745814;0.69972 +-0.50028425;0.99286544;-0.9937457;0.99993265;0.9744065;-0.10968865;0.23165157;0.9963624;-0.64632064;-0.92354345 +-0.99187315;0.8923065;-0.99882126;0.9926486;0.9904081;0.7033204;-0.3135256;0.32886347;-0.674517;0.95137817 +0.79290044;-0.9790674;-0.27615082;0.95422137;0.9967583;0.99954534;0.51418084;0.14904481;-0.8929588;0.2832205 +0.4455945;0.99065715;-0.9565973;0.9999041;0.9286983;-0.3742205;-0.59694415;0.99845976;-0.91822827;-0.90872437 +0.88152575;-0.986755;0.98323876;-0.9938217;0.1813723;0.9225639;0.7940811;-0.08075843;0.63817596;-0.9648439 +0.99852604;0.4048777;0.9892157;0.51243705;-0.8691542;0.22339989;-0.7637203;0.99060214;-0.09224681;-0.99806166 +0.99592865;0.45435935;0.90638375;0.94932544;0.101773925;0.93055266;-0.49095118;0.9961935;-0.96568733;-0.97288245 +0.9395518;-0.8398005;0.98902184;-0.9627669;-0.96114147;0.27439028;0.4405032;0.86634266;0.8607087;-0.9985272 +-0.8143577;0.9098442;-0.99287945;0.99925905;0.9954355;0.86075264;-0.23305719;0.9818527;-0.9133398;-0.26020685 +-0.786939;0.963124;-0.99197435;0.99955463;0.99141437;0.6446201;-0.40841058;0.9884821;-0.89324707;-0.45198295 +0.1385314;0.9918776;-0.9753591;0.99990904;0.9376721;-0.4176272;-0.63869405;0.9979067;-0.9044553;-0.8790141 +-0.99693286;0.9871457;-0.99967086;0.99767685;0.5345329;-0.9056782;0.2035433;-0.5232888;0.83130044;0.8860085 +-0.5218671;-0.9691465;-0.4238631;0.93389004;0.9870063;0.9994698;0.902434;-0.70915204;-0.6999531;0.3973869 +0.9941752;-0.24481036;0.91683555;0.7625688;0.45349342;0.9869857;-0.5407623;0.99225855;-0.9493281;-0.9740996 +-0.91257864;-0.979277;0.3979132;-0.5561029;0.95696104;0.999227;0.58844143;-0.98525244;-0.64424515;0.96956027 +-0.9697463;0.7975551;-0.99746627;0.99628776;0.99454355;0.86171675;-0.3453768;0.81608105;-0.8791955;0.8349014 +-0.16799086;-0.9869698;-0.6272001;0.71951425;0.9919162;0.99930066;-0.3934521;-0.7695571;-0.94823176;0.9567194 +-0.96781886;0.99568945;-0.9985012;0.9998472;0.9518045;-0.48152062;0.32121578;0.9689773;-0.07391523;-0.41372183 +0.2612673;0.96923804;-0.96618384;0.9998536;0.9777902;0.60521173;0.23172267;0.9984656;-0.8143277;-0.96108294 +-0.9956742;0.9895882;-0.99947846;0.999533;0.8761181;-0.701971;0.6996545;0.33156645;0.5889246;0.83528566 +-0.9964842;0.9900024;-0.99965256;0.99814266;0.45771644;-0.92017573;0.19018118;-0.41848183;0.8187314;0.86478376 +-0.9928524;-0.41904762;-0.99563444;0.8622692;0.9756168;0.96841687;-0.1763877;-0.73499954;0.010273453;0.975994 +0.9871693;0.80258596;0.66956043;0.9935921;0.6071478;0.8195179;-0.45517594;0.9985122;-0.9615796;-0.9709934 +0.9657787;0.9763003;-0.3241541;0.9993237;0.5928625;-0.66390556;-0.7902657;0.99890983;-0.96898764;-0.94546616 +-0.97000533;0.9954011;-0.99809253;0.99921864;0.835886;-0.76436865;-0.5748347;0.9261572;-0.17719847;0.53475296 +-0.8335399;-0.31923118;-0.99226433;0.98164916;0.99831337;0.9866125;-0.6517013;0.71019524;-0.9598605;0.92833984 +0.98962265;-0.48012343;0.98340124;-0.73626477;-0.9551022;-0.06750191;-0.64796346;0.964679;0.6297802;-0.9994089 +-0.99110454;0.9866474;-0.9990729;0.997861;0.9378635;-0.5361543;-0.2963164;0.51624656;-0.07949444;0.9213956 +-0.06519622;0.9948313;-0.9839185;0.99992794;0.95178527;0.027946671;0.7200171;0.997812;-0.26730603;-0.975428 +0.99875605;0.39920413;0.9913044;0.54431415;-0.8487416;0.17916285;-0.7269979;0.9902976;-0.34082687;-0.9976001 +-0.8993313;-0.9407501;-0.3613196;0.7107409;0.95518875;0.9991398;0.83514637;-0.9101761;-0.33380234;0.675788 +0.8815044;-0.9885791;0.985005;-0.99409044;0.33576113;0.93556416;0.80711627;-0.13483647;0.53332126;-0.9546784 +0.7554142;-0.9916319;0.6625426;0.7892409;0.9926285;0.99962854;0.19317175;-0.7088851;-0.9778166;0.83367515 +-0.15104565;-0.989883;-0.44646057;0.54247653;0.9907166;0.99931514;-0.44305667;-0.8604865;-0.95636374;0.9700833 +-0.9547437;0.7329617;-0.9976083;0.9995475;0.99474967;0.93070716;0.7189189;0.9271493;-0.6472043;-0.04998941 +-0.98320824;0.9941927;-0.9989131;0.99983424;0.9575278;-0.4282404;0.48985648;0.9416497;-0.09741292;-0.06328079 +-0.9979447;-0.4301556;-0.981381;0.40263742;0.73348147;0.91672456;0.47333854;-0.89882404;0.9175557;0.67514104 +0.9548418;-0.9917941;0.98191595;-0.97872823;0.9721907;0.99462384;-0.17627771;-0.08271227;-0.44315732;-0.4282902 +-0.7702768;-0.85931736;-0.9811983;0.98037314;0.9974149;0.99726593;-0.01230832;0.43473405;-0.92316496;0.8624166 +-0.99224013;0.9953382;-0.9995516;0.99956053;0.3849878;-0.9226922;0.48910707;0.46301135;0.7434338;0.6286314 +-0.9910794;0.9959111;-0.9994116;0.99981564;0.64374864;-0.86219954;0.7248646;0.6746193;0.6658838;0.50210077 +-0.9654552;0.9903211;-0.9978658;0.99982435;0.9804774;-0.00036579368;0.23818164;0.9696431;-0.5100166;-0.3103849 +0.99494857;0.725359;0.86746776;0.9819327;0.14849916;0.83465487;-0.52460504;0.99763626;-0.9693317;-0.9724266 +0.10691786;-0.39828342;-0.9518253;0.99664396;0.99500626;0.99479663;0.6536703;0.9681584;-0.6181241;-0.9121967 +-0.56157774;-0.9782544;-0.14246017;0.8252637;0.98655206;0.9995194;0.7948558;-0.8594559;-0.8485291;0.78796536 +-0.8051165;0.93522644;-0.99133515;0.9990533;0.9938024;0.78572893;-0.510548;0.98102325;-0.929799;-0.07666927 +-0.99693286;0.98818755;-0.999686;0.99735874;0.3663101;-0.9326675;0.053933024;-0.6030196;0.8715501;0.8689907 +-0.7934858;-0.9828768;-0.119498;0.10888173;0.98188806;0.9993334;0.36387318;-0.9634084;-0.85498905;0.9648024 +0.99766916;0.68554693;0.9266286;0.9801885;-0.41939512;0.7861951;-0.049959037;0.9967619;-0.9571291;-0.9869354 +0.97598195;0.9146251;0.0926689;0.9987744;0.52876806;0.70363414;0.74016243;0.99902296;-0.6057717;-0.9966486 +-0.9923372;0.9930486;-0.99928963;0.99978065;0.9342307;-0.5294078;0.5926337;0.82424486;0.056494217;0.51369953 +0.9646008;-0.9762558;0.98571783;-0.9918863;-0.19117048;0.91599494;0.16690151;0.7553328;0.47517082;-0.9889123 +-0.7982002;-0.9824467;0.65654516;-0.052571595;0.9704468;0.99953604;0.7707042;-0.9645243;-0.69749653;0.88354695 +0.879634;0.94429743;-0.54123104;0.99950093;0.9101184;0.4467465;-0.6294863;0.99867666;-0.9646537;-0.93443704 +-0.9913807;0.3726747;-0.9938201;0.9980261;0.9217152;0.90991646;0.9332079;0.32911706;0.5224765;-0.04046609 +-0.9884699;0.97922015;-0.9988246;0.9964079;0.97316015;-0.16610222;-0.5419426;0.56897354;-0.4646741;0.93891054 +0.97270024;-0.9631915;0.3615614;0.6407759;0.98926085;0.9990324;-0.060143147;0.83730197;-0.78521544;-0.76967454 +-0.29558453;0.9760004;-0.98190165;0.99972063;0.9806468;0.3782491;-0.6577849;0.99551445;-0.9218967;-0.78427607 +0.044846874;0.9969986;-0.9143487;0.9996262;0.7636535;-0.46625555;0.9178108;0.9965817;0.44245905;-0.99326885 +0.9175183;-0.95318604;0.9893595;-0.9887041;-0.83206725;0.7608956;0.5931291;0.7041904;0.8389282;-0.9948652 +-0.35897753;0.9945654;-0.9886171;0.99989635;0.9109617;-0.61510694;-0.7507972;0.99644715;-0.81859297;-0.81532085 +-0.87280756;0.9879347;-0.992989;0.9990499;0.963234;-0.011911978;-0.8206194;0.9766202;-0.8521642;0.13858624 +-0.7162032;0.9851742;-0.99389124;0.99986404;0.9852731;0.280254;-0.09204056;0.9941063;-0.76989204;-0.83587086 +-0.988733;0.9743442;-0.9986519;0.9981444;0.98434955;0.12798889;-0.29373384;0.71436167;-0.67377466;0.8967359 +-0.99260426;0.9810041;-0.9992426;0.9998131;0.96844685;0.13907063;0.8500989;0.8450453;0.21493943;0.0022677218 +-0.984144;0.9948657;-0.9988675;0.9993428;0.81699073;-0.8108348;-0.18407279;0.84993166;0.18276222;0.64797664 +0.9444956;0.9020631;-0.07720411;0.99886256;0.88216233;0.67240006;-0.43867007;0.9989024;-0.96091247;-0.9575145 +-0.90907973;0.9984118;-0.9981766;0.9998117;0.57114595;-0.9197376;-0.48233724;0.97286665;0.38383606;-0.4866659 +-0.9878949;-0.8461707;-0.94258904;0.44557464;0.93075514;0.99427617;0.38878822;-0.92663693;0.24014404;0.9293433 +-0.9920588;0.8095114;-0.9978728;0.99930376;0.9493619;0.7593545;0.9310102;0.6324554;0.41949105;0.17084312 +-0.92834187;0.9072137;-0.9963058;0.9995281;0.9956122;0.8547317;0.26861805;0.963969;-0.8082127;-0.14899516 +0.84895146;-0.99016875;0.95823187;-0.98657423;0.8779499;0.9867702;-0.21973534;-0.5279435;0.6145492;-0.7751291 +-0.9897106;0.4527174;-0.99840915;0.9967203;0.98756236;0.94160014;0.7342738;0.48233318;-0.22589159;0.74548525 +-0.9951544;0.97042334;-0.9991913;0.99957144;0.9600312;0.2657054;0.88094205;0.71357316;0.40678388;0.36920032 +0.87191033;0.96249276;-0.59583616;0.999582;0.8937472;0.17939062;-0.75687414;0.9985171;-0.97176504;-0.9151959 +0.8049074;-0.9317154;-0.5119577;0.9082369;0.9875133;0.99865186;0.6429125;0.61602277;-0.13749456;-0.84043324 +0.5909513;0.95156974;0.89661646;0.63199264;-0.95944804;-0.90738237;-0.30045584;0.98461604;0.97439337;-0.99955094 +0.98569024;-0.60258657;0.9832542;-0.92438716;-0.8064982;0.6713812;-0.7879805;0.9713566;0.6397578;-0.9988424 +-0.79426426;0.9619226;-0.9913248;0.9993634;0.9910543;0.6449043;-0.54935867;0.9860651;-0.9104378;-0.26984987 +-0.111089535;-0.43167707;-0.9784292;0.99570376;0.99846697;0.9961911;-0.046765573;0.96693003;-0.94356585;-0.2826349 +-0.9865993;0.9938036;-0.9989494;0.99925834;0.8619693;-0.76647705;-0.14401317;0.8093496;0.08816633;0.7362609 +-0.0551822;0.9981283;-0.9870968;0.9999263;0.6332305;-0.9086088;0.49878305;0.997575;-0.09149776;-0.95917565 +0.6735854;0.9742332;-0.8890006;0.9998593;0.947097;0.3511487;0.0724424;0.99905264;-0.8682248;-0.9699825 +-0.985018;0.99441403;-0.9988228;0.9996671;0.9141612;-0.657843;0.13103919;0.9053088;-0.12101398;0.46190882 +0.9814938;-0.96977395;0.9959642;-0.990972;-0.37529588;0.90117145;0.33437252;0.9063916;-0.043169815;-0.9904983 +-0.6556007;-0.94627285;-0.75755125;0.967156;0.9845735;0.99921495;0.9191151;-0.5465027;-0.40786865;0.043783132 +0.3402287;-0.959141;-0.8828496;0.9663798;0.99767005;0.99926466;-0.28478566;0.46040645;-0.9557238;0.6846941 +-0.97742;0.21753177;-0.9979184;0.99635136;0.9946849;0.97234833;0.5166852;0.68901104;-0.7125715;0.7494569 +0.99292076;0.77761513;0.96919614;0.91295314;-0.9440315;-0.9268555;0.3213481;0.9946752;-0.66998386;-0.99924743 +-0.65537065;0.9972593;-0.99674994;0.99992955;0.92786247;-0.687742;0.21609688;0.99425113;-0.23766656;-0.8785726 +-0.98522705;0.9877117;-0.99873984;0.9964152;0.9252493;-0.5747593;-0.7006891;0.5916057;-0.19224031;0.93313783 +-0.9949184;0.9929313;-0.99950063;0.9996411;0.7889831;-0.8059815;0.6892579;0.38081574;0.6087196;0.78817564 +0.7635552;-0.98424286;-0.38366675;0.8975194;0.9956842;0.9995246;-0.3804075;-0.16593876;-0.967952;0.8100838 +-0.50481933;-0.9512019;-0.9564744;0.97532856;0.9967756;0.99880207;0.121664144;0.14280495;-0.9059388;0.8029937 +0.1308624;0.9911992;-0.965631;0.9999036;0.9422619;0.33701178;0.8488757;0.998173;-0.15655595;-0.9861751 +0.9535762;0.9239645;0.0001449883;0.99895084;0.8281632;0.48742318;-0.5646966;0.998876;-0.9673419;-0.9536779 +-0.9939678;0.98930967;-0.99950343;0.9983133;0.77647716;-0.816833;-0.06669293;0.1253735;0.6153202;0.8984567 +-0.98005784;0.9929145;-0.998552;0.9979877;0.850949;-0.7532684;-0.67569447;0.78675467;-0.07352762;0.8413993 +0.9060802;0.006068822;-0.17027731;0.9828494;0.9688481;0.99105513;-0.30842233;0.9954642;-0.9155964;-0.9610857 +-0.9442235;0.99230164;-0.99668694;0.99956155;0.9627394;-0.21983045;-0.49534148;0.9723998;-0.7052965;0.10992921 +0.73230916;0.99260336;-0.9313838;0.9998039;0.7671724;-0.75996435;-0.8415675;0.9982094;-0.9213711;-0.89164907 +-0.9454959;-0.84986967;-0.9826101;0.84547776;0.9897458;0.9955344;-0.13280702;-0.5954892;-0.7496268;0.96728796 +0.2381937;-0.99162406;-0.12873645;0.7371342;0.99200714;0.99949604;-0.20011364;-0.8127806;-0.969278;0.9507532 +0.94069034;-0.991077;0.892025;0.3872416;0.9922192;0.9995881;0.25894302;-0.4693427;-0.9600582;0.3977052 +0.9949542;0.5809252;0.98541456;0.82197845;-0.9453901;-0.88260263;0.32626876;0.99054986;-0.6999937;-0.998981 +-0.9965385;0.9752922;-0.99951917;0.9992355;0.9344186;-0.36606753;0.7981995;0.2679716;0.67868185;0.7975869 +0.9529872;-0.98548794;0.98236287;-0.9934518;0.35706797;0.93711174;0.36243692;0.45734423;0.3943215;-0.97472566 +-0.6632041;-0.9788847;0.6284423;0.47755596;0.97769755;0.9995487;0.8706963;-0.9208286;-0.58009875;0.5833412 +-0.99163544;0.9788987;-0.99908805;0.99488646;0.93991363;-0.4484415;-0.5214022;0.16020666;-0.0050524194;0.9638261 +0.96964353;-0.60497904;0.39494273;0.8687129;0.90507716;0.9947968;0.41153792;0.9900666;-0.65402097;-0.98838675 +0.98178494;-0.9878964;0.9864816;-0.98621035;0.9459229;0.99086523;-0.08925618;0.53240377;-0.4887028;-0.7459057 +-0.9903756;-0.96061873;0.90890944;-0.97747046;-0.17165986;0.98489803;0.4939088;-0.99095845;0.94715726;0.51277757 +-0.010537528;-0.98842496;0.011978605;0.8615154;0.9918793;0.99960417;0.60254765;-0.8033734;-0.9440814;0.8689893 +-0.79087967;-0.9932079;0.98934174;-0.9898674;0.88190097;0.9951585;0.80945235;-0.98807114;0.42575502;0.35489208 +-0.44320706;-0.9846918;0.78998834;0.37506685;0.98376375;0.9995845;0.8689423;-0.9211994;-0.7037706;0.58226633 +-0.84979254;-0.51897407;-0.9935477;0.9961745;0.99806696;0.9935189;0.28837276;0.831465;-0.8936544;0.57455707 +0.60287905;0.99718237;-0.9203489;0.9998894;0.59630567;-0.8698872;0.8032114;0.9989212;-0.3665157;-0.99251145 +-0.7734865;0.78974885;-0.9927773;0.9988335;0.99717355;0.94235855;-0.11853391;0.97669715;-0.9252626;-0.13849288 +-0.8325051;0.89186764;-0.99214906;0.99833673;0.99520177;0.8496243;-0.5444399;0.9678049;-0.9414205;0.2844917 +0.6440773;-0.96819997;-0.7731626;0.955951;0.9973155;0.9994367;-0.23069902;0.39092758;-0.9465721;0.6020248 +-0.9979979;0.86075723;-0.99963194;0.9780901;0.90724814;-0.28067824;0.08545106;-0.77394843;0.84020567;0.9504507 +-0.97986734;0.99376947;-0.99875164;0.9998526;0.96411926;-0.3221594;0.52806836;0.9529303;-0.102227956;-0.27560502 +-0.9844988;0.9891885;-0.9986872;0.998196;0.94631994;-0.47872755;-0.5162223;0.7788971;-0.37720913;0.8655703 +-0.9891108;0.97519064;-0.99862725;0.99885875;0.9849599;0.16706394;-0.11488239;0.7792841;-0.6922748;0.8444137 +0.9161893;-0.9967324;0.9957328;-0.99621147;0.93366945;0.9883945;0.6928085;-0.67172366;-0.6038298;-0.4203235 +-0.95383674;-0.93189067;-0.5999943;0.4314518;0.9373052;0.9985297;0.67484605;-0.94854707;-0.2530151;0.8857599 +-0.91357005;0.95052487;-0.99430645;0.99756134;0.9900943;0.617256;-0.7580542;0.93391365;-0.9161346;0.6672664 +-0.97688055;-0.91758275;0.9029408;-0.9166051;-0.34220123;0.963327;0.45503378;-0.9722419;0.98633635;-0.6514135 +-0.88722044;0.9956099;-0.995969;0.9997381;0.9149708;-0.5641497;-0.68132615;0.98665464;-0.5971073;-0.4010819 +0.9919379;0.3591962;0.9824642;0.59168667;-0.9785623;-0.88805103;-0.05713873;0.9865598;-0.14280154;-0.99953735 +0.53578615;-0.99537396;0.9879111;-0.99182135;0.923353;0.9938117;0.4285336;-0.9323131;0.3639861;-0.3575771 +0.99789083;0.9096415;0.97134775;0.982485;-0.73689526;-0.71161497;-0.170281;0.99873775;-0.84174395;-0.9971906 +-0.9564074;-0.97615045;0.97014034;-0.9800486;0.45619565;0.9861735;0.73063093;-0.9839103;0.9334833;-0.14131294 +0.8927289;-0.9848804;0.40007767;0.87986135;0.99624574;0.99956065;0.51867306;0.016348194;-0.916941;0.18414348 +-0.9890525;-0.9200041;0.88457346;-0.9486072;-0.44354135;0.9687006;0.42767975;-0.98164535;0.98233795;-0.27115765 +-0.96994036;0.9014745;-0.9978721;0.9996466;0.99370384;0.8228105;0.59263736;0.9301544;-0.64908934;0.015207921 +-0.83760405;0.99803704;-0.9976593;0.99992174;0.86395425;-0.7710275;0.59244156;0.98533225;0.31772932;-0.8177556 +-0.80748355;0.9970797;-0.997566;0.99991727;0.9213403;-0.7092612;0.14310741;0.9912618;-0.15668368;-0.80971825 +-0.6764038;0.99191135;-0.99270076;0.9998797;0.9664684;-0.24547677;-0.5141019;0.99479246;-0.78829616;-0.7818368 +-0.4366161;0.996741;-0.994173;0.999935;0.94993097;-0.45469767;0.44955587;0.99640846;-0.3316535;-0.93635863 +0.84783685;-0.98867714;0.9721679;-0.99414223;0.60805625;0.95272416;0.38726246;-0.4035256;0.7166577;-0.919612 +0.7999439;0.97812057;-0.8184083;0.9998362;0.91198945;0.050319955;-0.18453415;0.9991246;-0.9075683;-0.9673632 +0.99574083;-0.7193864;0.99277484;-0.90582675;0.023770526;0.9634792;-0.78169435;0.9831605;-0.5143817;-0.99223787 +-0.93051517;-0.74563426;-0.9736616;0.98911756;0.98080397;0.99643546;0.874188;-0.07336275;-0.36595893;0.33132172 +0.06658689;-0.9945417;0.98664504;-0.99356127;0.62487876;0.97685;0.8646201;-0.9459542;0.8236547;-0.6132047 +-0.9919095;0.9955436;-0.99946165;0.9997825;0.66243094;-0.8609604;0.68651944;0.6210431;0.6533039;0.58237785 +-0.98176783;0.9969846;-0.9991825;0.9997976;0.6837351;-0.86775833;0.33025515;0.8832368;0.59749883;0.14116302 +-0.9820155;-0.874556;-0.9020235;0.47320336;0.93392235;0.9962823;0.5088932;-0.92975783;0.057884406;0.9191301 +-0.92760587;0.99607044;-0.99751127;0.9998389;0.9390977;-0.5435134;-0.21608436;0.9848253;-0.34511635;-0.5299455 +0.45135802;-0.99494904;0.9881855;-0.99442196;0.5467203;0.96579134;0.9033773;-0.8911597;0.7185843;-0.72937775 +0.88817453;-0.96937484;0.98200524;-0.9916615;-0.54136336;0.85692036;0.61355644;0.3758947;0.8468179;-0.9892876 +-0.94478226;0.9903399;-0.9965578;0.9996108;0.9750637;-0.045848258;-0.38324672;0.97328484;-0.73886526;0.025588254 +-0.9964833;0.9193339;-0.99931765;0.9963532;0.9814221;0.5191577;0.45257378;0.17935342;0.17822312;0.8934188 +-0.9734607;0.9983966;-0.9993268;0.99986243;0.22945406;-0.9466779;0.22379577;0.90542734;0.83765066;-0.43865153 +-0.87716824;-0.92749345;-0.4487588;0.84410477;0.95914143;0.99908984;0.8925541;-0.8498217;-0.1591106;0.33818573 +-0.75272757;-0.96822345;-0.5589084;0.8037379;0.9827585;0.9993235;0.7255603;-0.8518974;-0.76282954;0.81360555 +0.15165241;0.99646807;-0.9872087;0.9999369;0.8677382;-0.7774256;-0.5008502;0.99808615;-0.7486657;-0.91135776 +-0.98595226;0.9941903;-0.9988788;0.999558;0.8942111;-0.7098738;0.031498514;0.87440974;-0.038369495;0.59450215 +-0.9939948;0.9567428;-0.999;0.9962698;0.9824007;0.27024734;-0.10404067;0.3176401;-0.46626016;0.9494251 +-0.9888583;0.9965896;-0.99937046;0.9998321;0.59444815;-0.8763353;0.67393625;0.7587281;0.7053974;0.3290087 +0.24900626;-0.9117465;-0.9394346;0.98482186;0.99806756;0.99904335;-0.07378137;0.78030187;-0.9429418;0.3209028 +0.99726343;0.84698796;0.91753775;0.98938054;-0.38333526;0.39261094;-0.495885;0.9977919;-0.98038775;-0.9762415 +-0.9940278;0.988101;-0.99950033;0.9974578;0.73249424;-0.8328629;-0.2620754;-0.0649109;0.66302866;0.9173419 +0.40101352;0.9967798;-0.9774767;0.9999453;0.8433629;-0.76021713;0.18674919;0.99876994;-0.6132084;-0.95896333 +-0.56657284;0.97267425;-0.98880225;0.99968284;0.9870632;0.51358473;-0.5351687;0.9941052;-0.89564556;-0.732392 +-0.9964769;0.69671977;-0.99913824;0.9934048;0.9809969;0.82024217;0.53115207;-0.01732809;0.3633;0.8600272 +-0.6549648;0.5411401;-0.98989826;0.9966154;0.9980118;0.9713575;-0.39985442;0.96114707;-0.9582117;0.33980852 +-0.9549918;-0.58327687;-0.984871;0.9946773;0.9789776;0.9933803;0.8970562;0.1739004;-0.22701457;0.19718572 +0.9973301;-0.7211396;0.9979614;-0.7983251;-0.8829828;0.2633237;-0.15002996;0.9620156;-0.6457523;-0.9969337 +-0.99357736;0.9122085;-0.99902636;0.999163;0.98539084;0.6992834;0.64734006;0.7689523;-0.4121041;0.62574786 +0.9715105;0.983178;0.5792572;0.9941353;-0.8525414;-0.9623753;0.17913434;0.99884593;0.3141583;-0.9992369 +-0.9245339;0.98069537;-0.9960069;0.9997226;0.9892046;0.4013127;-0.10609029;0.97994894;-0.7698159;-0.35243368 +0.9969954;-0.83939594;0.9983702;-0.93596023;-0.7351173;0.693952;-0.26548746;0.9666108;-0.75882334;-0.9946644 +-0.7862518;0.98763883;-0.9947157;0.9998641;0.9833119;0.15884721;-0.17773075;0.9930545;-0.7437038;-0.78709424 +-0.9654533;0.9808242;-0.99728566;0.99952996;0.9882929;0.31718823;-0.15068401;0.952479;-0.7797815;0.25814977 +0.94204867;-0.9550606;0.9923048;-0.9880077;-0.82867396;0.75895834;0.61191314;0.7923919;0.7348045;-0.99516165 +0.989934;0.9775297;0.33874515;0.999465;0.18388304;-0.3492646;0.757042;0.9995371;-0.8023496;-0.99778634 +0.9982588;0.85009724;0.94322383;0.9895023;-0.5592154;0.3455683;0.1056926;0.9981503;-0.9670896;-0.99190027 +-0.47123206;-0.9872478;0.8370446;0.14340796;0.98384655;0.99960077;0.8468244;-0.9421827;-0.7679275;0.72998434 +-0.66769403;-0.98568404;0.7711244;0.04719726;0.97907776;0.9995907;0.81886494;-0.9549962;-0.7438734;0.82232153 +0.8931102;-0.9499758;0.94822174;-0.9903322;-0.49152339;0.8671167;-0.10328585;0.34783316;0.9346833;-0.9920549 +0.35392195;0.9237252;-0.95027316;0.99957377;0.9845614;0.79282635;-0.38686058;0.9977148;-0.9399567;-0.8921921 +-0.9948267;0.87528926;-0.9991126;0.99367994;0.9886141;0.75083274;0.08066408;0.2507484;-0.4119337;0.93562686 +-0.9729764;-0.97073776;0.9608201;-0.978692;0.26636428;0.985252;0.6841818;-0.9855501;0.94525725;-0.015171019 +-0.9715743;0.9979604;-0.9991889;0.9998199;0.53454995;-0.90928984;0.119364336;0.91301084;0.7245385;-0.20861752 +-0.30143097;-0.9958993;0.9916746;-0.98435557;0.89163285;0.9930376;0.8878629;-0.96939105;0.30008966;-0.19194439 +-0.01334633;-0.98117286;0.40226182;0.86875486;0.98996246;0.9996379;0.8953937;-0.7705024;-0.8009733;0.37115118 +-0.99538;0.99140567;-0.9994742;0.99969214;0.8780841;-0.68810534;0.76396066;0.45934013;0.5901155;0.7756407 +-0.39829704;-0.95662963;-0.91345376;0.9874245;0.99266696;0.99915504;0.8374087;-0.12579958;-0.58501095;0.1749631 +-0.98886794;0.9957617;-0.999453;0.9992186;0.39194778;-0.932073;-0.13543649;0.5722486;0.7380926;0.6378527 +-0.9764745;0.9940442;-0.998302;0.99937296;0.91621166;-0.622742;-0.37855664;0.92243606;-0.36886734;0.5826128 +-0.94716436;-0.12894675;-0.99635756;0.99861795;0.9949799;0.98657215;0.8202464;0.82532996;-0.57915616;0.2093027 +-0.98868996;0.99562544;-0.99926656;0.9996457;0.7170669;-0.8593594;0.2748974;0.7774395;0.51485795;0.5512841 +0.97202516;0.41057977;0.37533477;0.9816774;0.8218494;0.9764452;0.37017217;0.997125;-0.8018265;-0.99036396 +0.427549;-0.9929082;0.29057848;0.66213924;0.9918814;0.99955726;-0.14415862;-0.8508891;-0.9773754;0.9499447 +-0.99009097;0.99440736;-0.99929523;0.9994479;0.77215004;-0.83787376;0.15930647;0.68808436;0.44097164;0.72226435 +-0.9949338;0.9111132;-0.99912;0.99819267;0.98666245;0.70520943;0.45564133;0.61442643;-0.40862986;0.81601524 +-0.99646705;0.88351727;-0.9993301;0.9921559;0.9823521;0.5709544;0.25686353;-0.094094604;0.22274116;0.93527347 +-0.90416694;0.98595715;-0.994073;0.9975639;0.9566112;0.0025282523;-0.8919497;0.94710404;-0.84900355;0.5950615 +-0.8269473;-0.9948015;0.98686093;-0.9979346;0.8579633;0.9948882;0.7138748;-0.9968939;0.07014199;0.8686635 +0.83657265;-0.9783532;0.91153765;-0.98129886;0.6760429;0.9774666;-0.45017245;-0.32940468;0.8802422;-0.9275889 +-0.85275584;0.9826868;-0.9921945;0.99918985;0.9780899;0.23514675;-0.75515777;0.98095924;-0.87872124;-0.020921022 +-0.17194362;-0.9835607;-0.733822;0.8385831;0.9931139;0.9992906;-0.27314925;-0.6430792;-0.9401502;0.93514663 +-0.99363685;0.97677916;-0.998999;0.9992451;0.9794955;0.110901736;0.29920033;0.71560836;-0.49182904;0.8159747 +0.92521;-0.97121954;0.016550215;0.877405;0.9966661;0.9994903;-0.26888305;0.5796525;-0.92511046;0.10904193 +0.8667728;-0.98708415;0.5037789;0.8870981;0.9959066;0.9995958;0.51593137;-0.20670837;-0.94663435;0.415174 +-0.98436;0.9859571;-0.99889845;0.9998715;0.978221;0.15774502;0.7426926;0.9283597;-0.08083326;-0.342755 +0.9551747;0.097942285;0.3108415;0.97388333;0.925912;0.98744065;-0.48272824;0.99497217;-0.94046396;-0.9619527 +0.9070997;-0.86202306;-0.23113593;0.928296;0.99226856;0.9984955;-0.22669719;0.9382203;-0.83019644;-0.85538816 +0.9618002;-0.9933078;0.98765665;-0.9821549;0.98156375;0.9957526;-0.022195367;-0.21308671;-0.6799427;-0.15845832 +-0.88058555;0.6408498;-0.9945842;0.99302536;0.99664104;0.93366873;-0.65924996;0.8874848;-0.9439135;0.80983 +-0.97835505;0.8455264;-0.99775827;0.99375504;0.9936825;0.7988264;-0.5050538;0.6812815;-0.8589907;0.92177415 +-0.9090732;0.9910742;-0.99709743;0.9998879;0.980926;0.06489669;0.3178967;0.9869317;-0.43624985;-0.7781098 +-0.48930988;-0.98367065;0.77056485;0.39829013;0.9830763;0.9995655;0.87818146;-0.9176553;-0.6487795;0.53260815 +0.6363764;-0.9904878;-0.094433405;0.73895353;0.99272656;0.9994535;-0.5891906;-0.6716217;-0.97794497;0.9351503 +0.51273626;-0.98721397;-0.477572;0.81022286;0.99404156;0.9993851;-0.6296475;-0.5266426;-0.97020316;0.9261073 +-0.98188657;0.9913643;-0.9987669;0.99986136;0.97378576;-0.12249114;0.58784986;0.94635844;-0.18780166;-0.26195714 +-0.30076683;-0.9829986;0.35527602;0.7987016;0.98787177;0.9996288;0.81393003;-0.87287575;-0.8877737;0.76974523 +-0.9155738;0.98379004;-0.99434036;0.9967147;0.96004844;0.05373713;-0.90016955;0.9289422;-0.8522811;0.707973 +-0.12988016;0.9521528;-0.97863984;0.99961984;0.9886591;0.7133385;-0.42735064;0.996198;-0.93258774;-0.8311529 +0.8930526;-0.9937876;0.97752357;-0.92405844;0.977476;0.9977312;-0.3541463;-0.6162299;-0.43263394;-0.04126307 +-0.9883528;-0.7308601;-0.96274036;0.8264973;0.93616766;0.99330664;0.6191296;-0.8169872;0.22753148;0.8445488 +0.97381806;0.97332543;0.016581193;0.9996539;0.55061936;-0.35671952;-0.022182181;0.99932986;-0.95139354;-0.9806018 +0.25679928;-0.99002796;-0.3119624;0.8375183;0.9927371;0.9995012;-0.09200592;-0.72777987;-0.9619687;0.92651755 +-0.9969886;0.9596981;-0.999594;0.9973126;0.9283624;-0.42378855;0.5145;-0.18272813;0.6813575;0.9187862 +0.99783134;-0.6361281;0.99678874;-0.80266273;-0.8374334;0.49497926;-0.5431309;0.97230303;-0.63482827;-0.9968971 +0.78989327;-0.9896839;0.12646239;0.7981517;0.99350244;0.99953467;-0.47295314;-0.5228132;-0.98060113;0.8961549 +-0.3518241;-0.919762;-0.9749828;0.9937013;0.99728465;0.99873656;0.57122016;0.6064559;-0.8260068;0.21606249 +0.9154628;-0.99035037;0.9822458;-0.9945872;0.6885074;0.9562291;0.46734667;-0.08359578;0.35052654;-0.92734027 +-0.981581;0.990328;-0.99879736;0.9998733;0.97342145;-0.039410703;0.6802819;0.9445708;-0.06418845;-0.40582103 +-0.9884351;0.9811756;-0.9988373;0.9965969;0.9690089;-0.23854606;-0.53908116;0.5792225;-0.42258164;0.9358511 +0.96756244;-0.9812486;0.96126485;-0.9753597;0.9110696;0.99024564;-0.4659623;0.46113312;0.12012153;-0.8579779 +-0.9963045;0.97945833;-0.99935013;0.99953634;0.9280137;-0.14501092;0.90257674;0.49308684;0.7226866;0.47498953 +0.55497247;0.9814776;-0.91752577;0.99976695;0.9344962;-0.047362737;-0.8412616;0.9976258;-0.96398747;-0.83332443 +0.99839884;-0.042945143;0.9961286;0.09455337;-0.84838086;-0.17549428;-0.27289343;0.97186655;-0.7405427;-0.9966257 +-0.37503776;-0.9873943;0.6016178;0.47776;0.9869852;0.9996449;0.6270497;-0.94555724;-0.923448;0.91639453 +-0.99040806;0.99460614;-0.99925274;0.99964243;0.8261626;-0.79302555;0.33436406;0.7596474;0.34060678;0.6419426 +0.5549298;-0.99375266;0.6457241;0.4104793;0.9911035;0.9995629;-0.18817052;-0.9012193;-0.9806584;0.9521732 +-0.98651606;-0.968469;0.94223595;-0.98614186;0.056398347;0.98615754;0.6186172;-0.99215156;0.9310134;0.50220764 +-0.6594504;-0.13093475;-0.98999524;0.9930298;0.9986717;0.9886371;-0.4824447;0.9040846;-0.9640856;0.7115907 +0.9095627;-0.8003244;-0.21055962;0.93691146;0.98836446;0.9980117;-0.2777103;0.96554196;-0.8304983;-0.9118788 +-0.9897766;0.99511063;-0.99924755;0.99982756;0.91095054;-0.62888503;0.6372844;0.88419783;0.269012;0.19410971 +0.1931504;-0.9866382;0.7700656;0.6714327;0.988746;0.9996682;0.79640836;-0.8715294;-0.89294535;0.6451955 +0.46822506;-0.989468;0.9635635;-0.9927554;0.62231123;0.9731344;0.30613837;-0.8777559;0.908083;-0.8016054 +-0.99164206;0.9793924;-0.9990792;0.99566275;0.9513584;-0.3880221;-0.46297276;0.27758095;-0.10669073;0.95729715 +-0.98894536;0.85619396;-0.99867004;0.99947137;0.9839015;0.81072164;0.8396307;0.8315657;-0.2190615;0.28230494 +0.99758685;-0.32721972;0.99759847;-0.11322116;-0.8920387;-0.39899382;0.2719756;0.9597793;-0.7278217;-0.99677646 +-0.9902059;0.99492586;-0.99928784;0.99964184;0.78837115;-0.8220891;0.3321844;0.7487602;0.42483172;0.62575775 +0.8679997;-0.964759;0.97962224;-0.99152535;-0.5971184;0.84293914;0.5962257;0.31164616;0.8882343;-0.9898251 +-0.97632444;0.9797416;-0.99800694;0.99668247;0.97647226;0.008529108;-0.7228869;0.79224205;-0.717813;0.8930228 +0.9259375;-0.9464377;0.9904701;-0.98752266;-0.86442083;0.7245895;0.58455783;0.7576783;0.82773495;-0.9956604 +-0.75588644;-0.47308406;-0.9931762;0.9983159;0.99780345;0.9943854;0.59282875;0.91308045;-0.83382887;-0.07641294 +0.98528147;-0.9646924;0.9545325;-0.97501326;0.7919693;0.9856697;-0.6370326;0.82879305;0.17707177;-0.9541144 +0.9550027;-0.98728454;0.9712262;-0.9755401;0.9491953;0.9931694;-0.368639;0.16548212;-0.06468185;-0.70155114 +-0.9917377;0.982681;-0.99909556;0.99681425;0.94533646;-0.45809916;-0.3746823;0.36806372;-0.07618672;0.9453995 +-0.2539207;0.9982604;-0.99102277;0.99992627;0.6386554;-0.9090581;0.43829456;0.9967493;0.034954578;-0.9462662 +0.9945277;-0.13654056;0.8792574;0.8153112;0.40802875;0.98160434;0.27136007;0.99578565;-0.8207325;-0.9936132 +-0.9878102;0.9969081;-0.9992844;0.9998687;0.67155415;-0.84598285;0.77602136;0.8070883;0.6962646;0.19406855 +0.9928409;-0.93737227;0.9771613;-0.97990435;0.6254731;0.9805373;-0.67889893;0.94230515;-0.14003493;-0.9772076 +-0.9611551;0.9773079;-0.9970761;0.9995557;0.9899633;0.41534674;-0.10220436;0.9571976;-0.78660357;0.155522 +0.92697304;-0.8379502;-0.3489263;0.8955106;0.96930563;0.99736977;0.46869358;0.9434393;-0.3353937;-0.96443945 +0.99313504;0.92700523;0.7230611;0.99829686;0.016459104;0.3606415;0.4512433;0.9991004;-0.92086744;-0.99363995 +-0.9631071;0.9971116;-0.998538;0.99980927;0.8967638;-0.722819;0.062150244;0.9650654;0.118112504;-0.29895774 +0.99649125;0.833227;0.8932291;0.99146634;-0.25855148;0.61598045;-0.22766739;0.99792653;-0.96762043;-0.9831726 +-0.9546354;-0.64958084;-0.99222904;0.9755037;0.99346787;0.99243176;0.25536978;0.058971863;-0.7492685;0.9099184 +0.9461333;-0.78204465;0.987554;-0.94033986;-0.9705307;0.07523222;0.38487008;0.88559276;0.86754465;-0.9988826 +0.99658424;0.9065405;0.86113495;0.9964201;-0.26365277;0.31621918;0.3970626;0.99888545;-0.9425887;-0.9943862 +0.998445;-0.20276676;0.994847;-0.35216555;-0.8612202;0.27505165;-0.6531059;0.97818685;-0.5248854;-0.99741656 +0.86075634;-0.98860115;0.6411871;0.8536799;0.9951447;0.99961406;0.47667044;-0.37099287;-0.95983684;0.50861716 +-0.9221573;0.9919899;-0.99597585;0.9996382;0.96487445;-0.17548294;-0.5308392;0.9807367;-0.72949946;-0.13989766 +0.9003411;0.8406733;-0.3739291;0.9985658;0.9481254;0.85532224;-0.34902;0.99867517;-0.9525439;-0.95568603 +0.9960321;0.22094947;0.9922478;0.6084953;-0.9491768;-0.78776854;0.24388823;0.9820401;-0.6226409;-0.9987349 +0.843049;-0.9908215;0.97556293;-0.99420583;0.6896985;0.96050626;0.433493;-0.49684957;0.6473776;-0.8888805 +0.13104425;-0.98496115;0.69822824;0.7413814;0.9890188;0.9996603;0.83730567;-0.8470659;-0.86475474;0.55607086 +-0.78965044;0.97044593;-0.9945418;0.9998062;0.9909752;0.63141686;0.19635163;0.9910537;-0.749107;-0.8148882 +0.9985273;-0.36975953;0.97752476;0.4741367;-0.3814138;0.9764832;-0.5594176;0.98910975;-0.9718568;-0.97848946 +-0.99475515;0.9624857;-0.99905753;0.99848104;0.98235446;0.32266742;0.24575967;0.56830245;-0.44805288;0.8801853 +0.98263365;0.95856243;0.32518062;0.9994431;0.43902275;0.14461847;0.41781378;0.99935806;-0.9157144;-0.9904469 +0.9923904;0.9245943;0.9022452;0.9829599;-0.90176004;-0.9492602;0.49134403;0.9978941;-0.69663244;-0.9990062 +-0.9752078;-0.11257827;-0.99714464;0.99169785;0.9947924;0.981464;0.4116065;0.45760107;-0.70657706;0.8595093 +0.9918902;0.09439927;0.8708363;0.902203;0.5608401;0.9797007;-0.549198;0.99431163;-0.9577581;-0.9682778 +0.99816936;-0.010257589;0.99488777;0.11017133;-0.8860679;-0.2465429;-0.41651237;0.9743409;-0.6244988;-0.9976782 +-0.47426564;0.3087743;-0.9903675;0.9977755;0.9983085;0.9851718;-0.12562762;0.9753092;-0.94511;-0.1840757 +-0.7515603;-0.9865488;0.7662461;-0.23545657;0.9756285;0.9995665;0.77271825;-0.96961904;-0.7661363;0.9046627 +-0.99013156;0.99652356;-0.99951303;0.9996326;0.08328268;-0.9496029;0.25613007;0.59432805;0.81777143;0.35314575 +-0.9952058;0.96722364;-0.9992039;0.9966474;0.9599457;-0.19529277;0.12848595;0.057471756;0.053963296;0.9556676 +-0.86829716;-0.9591594;0.013055038;0.56709635;0.95966166;0.9993545;0.82116896;-0.9328605;-0.4420747;0.7444097 +-0.6471375;-0.9950024;0.99232167;-0.98855036;0.9177832;0.9959536;0.8450752;-0.98644304;0.17353503;0.36515084 +0.32972553;-0.9582115;-0.8849516;0.96635574;0.9976901;0.9992533;-0.29671693;0.4694001;-0.9568736;0.68919516 +0.97518456;-0.60260254;0.62654895;0.82764536;0.9167015;0.99578166;-0.36630404;0.9899085;-0.88516486;-0.9728662 +-0.99276453;0.96181184;-0.99908566;0.99964386;0.98423517;0.42948928;0.64009607;0.8333522;-0.38607162;0.45815945 +0.99392986;0.8383213;0.9486269;0.8532042;-0.9232351;-0.79026365;-0.58927757;0.99686515;-0.123654775;-0.99953467 +0.8334069;0.9455451;-0.6783608;0.9995973;0.9336121;0.5082;-0.5615718;0.99868387;-0.9581422;-0.93466824 +-0.99397314;0.98886645;-0.99942;0.9998254;0.9580269;-0.17148037;0.8046557;0.8336498;0.23475823;0.22498254 +0.1033773;0.9891302;-0.97884893;0.9999189;0.96904945;0.3098591;0.55891573;0.9983864;-0.57643986;-0.9760518 +-0.991869;-0.9498998;0.8637446;-0.96530443;-0.29758155;0.983168;0.41658318;-0.989249;0.955648;0.48556697 +-0.99571884;0.9919425;-0.9996169;0.9983501;0.34518793;-0.9328274;0.10702716;-0.3152972;0.80613923;0.8444992 +-0.9907954;-0.9369117;0.8896722;-0.9601975;-0.37931624;0.97549796;0.46038842;-0.9852802;0.9741431;0.03856493 +0.97307837;-0.984817;0.84787124;0.004105895;0.9907279;0.9994007;0.14922336;0.3734007;-0.86289304;-0.34127966 +0.73435163;-0.98605394;-0.30469376;0.8583167;0.9947655;0.9994999;-0.5223253;-0.33258176;-0.9735129;0.867345 +0.99376994;0.79308397;0.9561777;0.7705433;-0.9397059;-0.73250824;-0.71222943;0.9960673;0.15659669;-0.99957645 +0.96178955;-0.9788519;0.5404554;0.5741475;0.9932275;0.99942756;0.19159366;0.5179758;-0.8248911;-0.51132417 +-0.8022166;-0.7202056;-0.9898247;0.9936079;0.9981111;0.99603635;0.23109561;0.7589013;-0.90732056;0.66730654 +0.91857994;0.97567564;-0.6189021;0.9998205;0.8333651;0.19010606;0.58391535;0.99950206;-0.8084022;-0.9886378 +0.8149848;-0.98732954;-0.05878605;0.85953194;0.9949161;0.9995505;-0.39713475;-0.30831513;-0.9760073;0.8455541 +-0.9641019;0.9469436;-0.9971451;0.9990173;0.9928456;0.633879;-0.23332164;0.92306083;-0.85421085;0.5707248 +0.9950737;-0.8782805;0.99821615;-0.9417042;-0.8314765;0.58679664;0.12858623;0.9582659;-0.57117957;-0.9962673 +-0.9234458;-0.9773858;0.97284126;-0.9595446;0.5327261;0.9848346;0.7713445;-0.9729487;0.9317996;-0.5205868 +-0.95910925;0.4431598;-0.99749535;0.9992506;0.9949314;0.9674837;0.7837236;0.8939928;-0.616433;0.11639371 +0.89977115;0.9843275;-0.7501627;0.99987566;0.82988006;-0.2828893;0.22948816;0.99942404;-0.8711139;-0.98004586 +-0.1557516;0.987853;-0.9840808;0.99991006;0.9737038;0.04570853;-0.21239847;0.99751633;-0.86414415;-0.90859884 +-0.8035597;-0.87885875;-0.9783908;0.9575391;0.9966548;0.99692374;-0.22630475;0.09781378;-0.9226663;0.93415326 +0.71884096;0.99574834;-0.94267845;0.9999291;0.7477629;-0.80872643;0.17613152;0.99909985;-0.75396997;-0.9690089 +-0.9779125;-0.97110826;0.95623255;-0.97952384;0.22203267;0.98660946;0.6582727;-0.9873102;0.9376967;0.14342013 +0.28480262;-0.9886843;-0.42719683;0.8833167;0.99330187;0.9995;-0.019455831;-0.64811015;-0.9552309;0.9012979 +-0.9875795;0.9934884;-0.99894375;0.99970275;0.9368304;-0.564176;0.2625677;0.8920809;-0.19820416;0.49458626 \ No newline at end of file diff --git a/project/housQR.jl b/project/housQR.jl deleted file mode 100644 index 9f1e31b..0000000 --- a/project/housQR.jl +++ /dev/null @@ -1,135 +0,0 @@ -module housQR - -import Base: size, show, getproperty, getfield, propertynames, \, * -using LinearAlgebra: norm, I, triu, diagm, dot - -export QRhous, qrfact - -mutable struct QRhous{T <: Real} - A::AbstractVecOrMat{T} - d::AbstractArray{T} - AQ::Union{Nothing, AbstractVecOrMat{T}} - AR::Union{Nothing, AbstractVecOrMat{T}} - - QRhous(A, d, AQ=nothing, AR=nothing) = new{eltype(A)}(A, d, AQ, AR) -end - - - -function householder_vector(x::AbstractVecOrMat{T})::Tuple{Vector{T}, T} where T - # returns the normalized vector u such that H*x is a multiple of e_1 - s = norm(x) - if x[1] ≥ 0 - s = -s - end - u = copy(x) - u[1] -= s - u ./= norm(u) - return u, s -end - -function qrfact(A::Matrix{T})::QRhous where T - (m, n) = size(A) - R = deepcopy(A) - d = zeros(n) - - for k ∈ 1:min(m, n) - @views (u, s) = householder_vector(R[k:m, k]) - # construct R - d[k] = s - @views R[k:m, k+1:n] -= 2 * u * (u' * R[k:m, k+1:n]) - R[k:m, k] .= u - end - return QRhous(R, d) -end - - -function qyhous(A::QRhous{T}, y::AbstractArray{T}) where T - m, n = size(A) - z = deepcopy(y) - for k ∈ n:-1:1 - # z[j:m] = z[j:m] - 2 * A.A[j:m, j] * (A.A[j:m, j]' * z[j:m]) - @views z[k:m] -= 2 * A.A[k:m, k] .* dot(A.A[k:m, k], z[k:m]) - end - return z -end - -function qyhoust(A::QRhous{T}, y::AbstractArray{T}) where T - m, n = size(A) - z = deepcopy(y) - for k ∈ 1:n - # z[j:m] = z[j:m] - 2 * A.A[j:m, j] * (A.A[j:m, j]' * z[j:m]) - @views z[k:m] -= 2 * A.A[k:m, k] .* dot(A.A[k:m, k], z[k:m]) - end - return z -end - -function calculateQ(A::QRhous{T}) where T - if A.AQ != nothing - return A.AQ - end - m, n = size(A.A) - - A.AQ = zeros(m, 0) - id = Matrix{eltype(A.A)}(I, m, n) - for i ∈ eachcol(id) - A.AQ = [A.AQ qyhous(A, i)] - end - return A.AQ -end - -function calculateR(A::QRhous{T}) where T - if A.AR != nothing - return A.AR - end - m, n = size(A.A) - A.AR = triu(A.A[1:n, :], 1) + diagm(A.d) - return A.AR -end - -function Base.show(io::IO, mime::MIME{Symbol("text/plain")}, A::QRhous{T}) where T - summary(io, A); println(io) - print(io, "Q factor: ") - show(io, mime, A.Q) - print(io, "\nR factor: ") - show(io, mime, A.R) -end - -function Base.getproperty(A::QRhous{T}, d::Symbol) where T - if d === :R - return calculateR(A) - elseif d === :Q - return calculateQ(A) - else - getfield(A, d) - end -end - -Base.propertynames(A::QRhous, private::Bool=false) = (:R, :Q, (private ? fieldnames(typeof(A)) : ())...) - -Base.size(A::QRhous) = size(getfield(A, :A)) - -function (\)(A::QRhous{T}, b::AbstractVector{T}) where T - n, m = size(A) - v = qyhoust(A, b) - x = zeros(m) - - - @show size(v) - @show size(x) - @show size(A.A) - @show size(A.d) - @show m - #@inbounds begin - for j ∈ m:-1:1 - @views x[j] = (v[j] - dot(x[j+1:m], A.A[j, j+1:m])) * A.d[j]^-1 - end - #end - return x -end - -function (*)(A::QRhous{T}, x::AbstractVecOrMat{T}) where T - return qyhous(A, (A.R * x)) -end - -end \ No newline at end of file diff --git a/project/plotting/L-BFGS.py b/project/plotting/L-BFGS.py new file mode 100644 index 0000000..07520c7 --- /dev/null +++ b/project/plotting/L-BFGS.py @@ -0,0 +1,428 @@ +from config import ( + LBFGS_RESULTS, + PLOTS_PATH, + log_tick_formatter, + nanoseconds_to_seconds, + setup +) + +from scipy.interpolate import griddata +from mpl_toolkits.mplot3d import Axes3D +import pandas as pd +import matplotlib.pyplot as plt +import numpy as np +from matplotlib import ticker, cm +import seaborn as sns +from scipy.interpolate import interp1d + +metrics_map = { + 'error': ['relative', 'residual'], + 'time': ['meantime'], + 'iterations': ['iterations'] +} + + +def plot_lambda_epsilon_3D(kind): + if kind not in ['error', 'time', 'iterations']: + raise ValueError("kind must be one of 'error', 'time', 'iterations'") + + df = pd.read_csv(LBFGS_RESULTS.format("3D", "statisticsLBFGS-lambda-eps-m300n20--{}.csv".format(kind))) + + metrics = metrics_map[kind] + + for metric in metrics: + + lambda_values = df['lambda'] + epsilon_values = df['epsilon'] + meantime_values = df[metric] + + lambda_mesh, epsilon_mesh = np.meshgrid(np.unique(lambda_values), np.unique(epsilon_values)) + + # Cubic interpolation of the data to make it smooth + meantime_interp = griddata((lambda_values, epsilon_values), meantime_values, (lambda_mesh, epsilon_mesh), + method='cubic') + + # Plot the 3D surface + fig = plt.figure(figsize=(13, 9)) + ax = fig.add_subplot(111, projection='3d') + + sns.set(style="white", context="paper") + + surface = ax.plot_surface(np.log10(lambda_mesh), np.log10(epsilon_mesh), meantime_interp, edgecolor='none', + cmap='viridis', alpha=0.9) + + fig.colorbar(surface, ax=ax, shrink=0.5, aspect=5, pad=0.2) + + # Set labels + ax.set_xlabel('λ', labelpad=20, fontsize=15) + ax.set_ylabel('ϵ', labelpad=20, fontsize=15) + + if metric == 'meantime': + metric = 'running time' + ax.set_zlabel('time (s)', labelpad=10, fontsize=20) + ax.zaxis.set_major_formatter(ticker.FuncFormatter(nanoseconds_to_seconds)) + title = 'L-BFGS ' + metric + ' with respect to λ and ϵ over 10000 runs' + elif metric in ['relative', 'residual']: + if metric == 'relative': + metric = 'relative error' + ax.set_zlabel(metric, labelpad=10, fontsize=20) + title = 'L-BFGS ' + metric + ' with respect to λ and ϵ' + else: + ax.set_zlabel(metric, labelpad=10, fontsize=20) + title = 'L-BFGS ' + metric + ' with respect to λ and ϵ' + + plt.title(title, fontsize=20, pad=20, fontweight='bold', loc='center') + + # Set logarithmic locator of ticks + ax.xaxis.set_major_formatter(ticker.FuncFormatter(log_tick_formatter)) + ax.yaxis.set_major_formatter(ticker.FuncFormatter(log_tick_formatter)) + + ax.xaxis.set_tick_params(size=15) + ax.yaxis.set_tick_params(size=15) + ax.zaxis.set_tick_params(size=15) + + metric = metric.replace(" ", "_") + plt.savefig(PLOTS_PATH.format("LBFGS/3D", "LBFGS-lambda-eps--{}.png".format(metric))) + plt.show() + + +def plot_epsilon_memory_3D(kind): + if kind not in ['error', 'iterations']: + raise ValueError("kind must be one of 'error', 'iterations'") + + df = pd.read_csv(LBFGS_RESULTS.format("3D", "statisticsLBFGS-eps-mem-m300n20--{}.csv".format(kind))) + + metrics = metrics_map[kind] + + for metric in metrics: + + memory_values = df['memsize'] + epsilon_values = df['epsilon'] + meantime_values = df[metric] + + memory_mesh, epsilon_mesh = np.meshgrid(np.unique(memory_values), np.unique(epsilon_values)) + + # Cubic interpolation of the data to make it smooth + meantime_interp = griddata((memory_values, epsilon_values), meantime_values, (memory_mesh, epsilon_mesh), + method='cubic') + + # Plot the 3D surface + fig = plt.figure(figsize=(13, 9)) + ax = fig.add_subplot(111, projection='3d') + + sns.set(style="white") + sns.set_context('paper') + + surface = ax.plot_surface(memory_mesh, np.log10(epsilon_mesh), meantime_interp, edgecolor='none', + cmap='viridis', alpha=0.9) + + fig.colorbar(surface, ax=ax, shrink=0.5, aspect=5, pad=0.2) + + # Set labels + ax.set_xlabel('memory size', labelpad=20, fontsize=15) + ax.set_ylabel('ϵ', labelpad=20, fontsize=15) + ax.set_zlabel('iterations', labelpad=10, fontsize=20) + + if metric in ['relative', 'residual']: + if metric == 'relative': + metric = 'relative error' + ax.set_zlabel(metric, labelpad=10, fontsize=20) + + plt.title('L-BFGS ' + metric + ' with respect to the memory size and ϵ', fontsize=20, pad=20, + fontweight='bold', + loc='center') + + ax.yaxis.set_major_formatter(ticker.FuncFormatter(log_tick_formatter)) + + ax.xaxis.set_tick_params(size=15) + ax.yaxis.set_tick_params(size=15) + ax.zaxis.set_tick_params(size=15) + plt.savefig(PLOTS_PATH.format("LBFGS/3D", "LBFGS-eps-mem--{}.png".format(metric))) + plt.show() + + +def plot_iterations_error_2D(conditioning="well"): + if conditioning not in ["well", "ill"]: + raise ValueError("conditioning must be one of 'well' or 'ill'") + + setup( + title=f'L-BFGS errors with respect to iterations on {conditioning}-conditioned matrix', + x_label='Iterations', + y_label='Error' + ) + conditioning = conditioning + "_conditioned" + + df = pd.read_csv(LBFGS_RESULTS.format(conditioning, "statisticsLBFGS-iterations-m1000n20--error.csv")) + + maxiterations_values = df['maxiterations'] + relative_values = df['relative'] + residual_values = df['residual'] + + # since from a certain row the residual and the relative gap are the same as the previous row, we can all the + # next rows and keep only the first row of each value + relative_values = relative_values.drop_duplicates(keep='first') + residual_values = residual_values.drop_duplicates(keep='first') + + assert len(relative_values) == len(residual_values) + + maxiterations_values = maxiterations_values[:len(relative_values)] + + max_iter_interp = np.linspace(min(maxiterations_values), max(maxiterations_values), 100) + + relative_interp = interp1d(maxiterations_values, relative_values, kind='cubic')(max_iter_interp) + residual_interp = interp1d(maxiterations_values, residual_values, kind='cubic')(max_iter_interp) + + data = pd.DataFrame({ + 'maxiterations': max_iter_interp, + 'relative': relative_interp, + 'residual': residual_interp + }) + + sns.lineplot(data=data, x='maxiterations', y='relative', label='Relative Error', color='blue') + sns.lineplot(data=data, x='maxiterations', y='residual', label='Residual', color='red') + + plt.legend(loc='upper right', fontsize=15) + plt.ylim(0, None) + + conditioning = "LBFGS/" + conditioning + + plt.savefig(PLOTS_PATH.format(conditioning, "LBFGS-iterations-error.png")) + plt.show() + + +def plot_norm_gradient_2D(conditioning="well", smoothing_window=5): + if conditioning not in ["well", "ill"]: + raise ValueError("conditioning must be one of 'well' or 'ill'") + cond = conditioning + setup( + title=f'L-BFGS convergence on {cond}-conditioned matrix', + x_label='Iterations', + y_label=' ' + ) + + conditioning = conditioning + "_conditioned" + + df = pd.read_csv(LBFGS_RESULTS.format(conditioning, "statisticsLBFGS-iterations-m1000n20--error-norm.csv")) + + gradnorm = df['mean_gradient'].unique() + error = df['mean_relative'].unique() + residual = df['mean_residual'].unique() + + iterations = list(range(len(gradnorm))) + + assert len(error) == len(residual) == len(gradnorm) + + iterations_interp = np.linspace(min(iterations), max(iterations), len(residual)) + gradnorm = interp1d(iterations, gradnorm, kind='quadratic')(iterations_interp) + error = interp1d(iterations, error, kind='quadratic')(iterations_interp) + residual = interp1d(iterations, residual, kind='quadratic')(iterations_interp) + + smoothed_gradnorm = np.convolve(gradnorm, np.ones(smoothing_window) / smoothing_window, mode='valid') + smoothed_error = np.convolve(error, np.ones(smoothing_window) / smoothing_window, mode='valid') + smoothed_residual = np.convolve(residual, np.ones(smoothing_window) / smoothing_window, mode='valid') + + data = pd.DataFrame({ + 'iterations': iterations[:len(smoothed_gradnorm)], + 'Gradient Norm': smoothed_gradnorm, + 'Relative Error': smoothed_error, + 'Residual': smoothed_residual + }) + + sns.lineplot(data=data, x='iterations', y='Gradient Norm', label='Gradient Norm', color='green') + sns.lineplot(data=data, x='iterations', y='Relative Error', label='Relative Error', color='blue') + sns.lineplot(data=data, x='iterations', y='Residual', label='Residual', color='red') + + plt.yscale('log') + plt.legend(loc='upper right', fontsize=15) + + conditioning = "LBFGS/" + conditioning + plt.savefig(PLOTS_PATH.format(conditioning, f"LBFGS-iterations-gradient-{cond}.png")) + plt.show() + + +def plot_iteration_memory(conditioning="well", smoothing_window=3): + if conditioning not in ["well", "ill"]: + raise ValueError("conditioning must be one of 'well' or 'ill'") + + cond = conditioning + setup(title=f'L-BFGS relative error for different memory sizes on {cond}-conditioned matrix', x_label='Iterations', y_label='Relative Error') + + conditioning = conditioning + "_conditioned" + + df = pd.read_csv( + LBFGS_RESULTS.format(conditioning, "statisticsLBFGS-iterations-m1000n20--memsize.csv"), + ) + + unique_memsize = df['memsize'].unique() + + for memsize in unique_memsize: + data = df[df['memsize'] == memsize] + data.reset_index(drop=True, inplace=True) + + gradients_indices = data.index[data['gradient'].drop_duplicates(keep='first').index] + relative_errors = data.loc[gradients_indices]['relative'].values + + extension = np.full(smoothing_window, relative_errors[-1]) + relative_errors = np.concatenate((relative_errors, extension)) + + print(len(relative_errors)) + + iterations = list(range(len(relative_errors))) + iterations_interp = np.linspace(min(iterations), max(iterations), len(relative_errors)) + relative_interp = interp1d(iterations, relative_errors, kind='cubic')(iterations_interp) + + smoothed_relative = np.convolve(relative_interp, np.ones(smoothing_window) / smoothing_window, mode='valid') + + data = pd.DataFrame({ + 'iterations': iterations_interp[:len(smoothed_relative)], + 'relative': smoothed_relative + }) + + sns.lineplot(data=data, x='iterations', y='relative', label='Memory Size = {}'.format(memsize)) + + plt.legend(loc='upper right', fontsize=15) + plt.yscale('log') + + conditioning = "LBFGS/" + conditioning + plt.savefig(PLOTS_PATH.format(conditioning, f"LBFGS-iterations-memory-{cond}.png")) + plt.show() + + +def plot_awls_vs_exact_gradient_decrease_2D(smoothing_window=3): + conditioning = "well_conditioned" + df2_awls_quad = pd.read_csv( + LBFGS_RESULTS.format(conditioning, "statisticsLBFGS-AWLS-quad-iterations-m1000n20--error-norm.csv"), + ) + df3_exact = pd.read_csv( + LBFGS_RESULTS.format(conditioning, "statisticsLBFGS-iterations-m1000n20--error-norm.csv"), + ) + + setup(title='L-BFGS ExactLS vs AWLS gradient norm on well-conditioned matrix', x_label='Iterations', y_label='Gradient Norm') + + gradnorm_awls_quad = df2_awls_quad['mean_gradient'].unique() + gradnorm_exact = df3_exact['mean_gradient'].unique() + + extension = np.full(len(gradnorm_awls_quad) - len(gradnorm_exact) + smoothing_window, gradnorm_exact[-1]) + gradnorm_exact = np.concatenate((gradnorm_exact, extension)) + + extension = np.full(smoothing_window, gradnorm_awls_quad[-1]) + gradnorm_awls_quad = np.concatenate((gradnorm_awls_quad, extension)) + + assert len(gradnorm_awls_quad) == len(gradnorm_exact) + + iterations = list(range(len(gradnorm_awls_quad))) + + iterations_interp = np.linspace(min(iterations), max(iterations), len(gradnorm_awls_quad)) + + gradnorm_awls_quad = interp1d(iterations, gradnorm_awls_quad, kind='cubic')(iterations_interp) + gradnorm_exact = interp1d(iterations, gradnorm_exact, kind='cubic')(iterations_interp) + + smoothed_gradnorm_awls_quad = np.convolve(gradnorm_awls_quad, np.ones(smoothing_window) / smoothing_window, + mode='valid') + smoothed_gradnorm_exact = np.convolve(gradnorm_exact, np.ones(smoothing_window) / smoothing_window, mode='valid') + + data = pd.DataFrame({ + 'iterations': iterations_interp[:len(smoothed_gradnorm_awls_quad)], + 'AWLS quadratic': smoothed_gradnorm_awls_quad, + 'ExactLS': smoothed_gradnorm_exact + }) + + sns.lineplot(data=data, x='iterations', y='AWLS quadratic', label='AWLS', color='blue') + sns.lineplot(data=data, x='iterations', y='ExactLS', label='ExactLS', color='green') + + plt.yscale('log') + plt.legend(loc='upper right', fontsize=15) + + conditioning = "LBFGS/" + conditioning + plt.savefig(PLOTS_PATH.format(conditioning, "LBFGS-LS-gradient-comparison.png")) + plt.show() + + +def BFGS_comparison(): + folder = "comparison_BFGS" + df_BFGS = pd.read_csv( + LBFGS_RESULTS.format(folder, "statisticsBFGS-iterations-m1000n20--error-norm.csv"), + ) + df_LBFGS = pd.read_csv( + LBFGS_RESULTS.format("well_conditioned", "statisticsLBFGS-iterations-m1000n20--error-norm.csv"), + ) + + setup(title='BFGS vs L-BFGS gradient norm on well-conditioned matrix', x_label='Iterations', y_label=' ') + + gradnorm_BFGS = df_BFGS['mean_gradient'][::2] + gradnorm_LBFGS = df_LBFGS['mean_gradient'][::2] + error_BFGS = df_BFGS['mean_relative'][::2] + error_LBFGS = df_LBFGS['mean_relative'][::2] + residual_BFGS = df_BFGS['mean_residual'][::2] + residual_LBFGS = df_LBFGS['mean_residual'][::2] + + min_len = 27 + + iterations = list(range(min_len)) + gradnorm_BFGS = gradnorm_BFGS[:min_len] + gradnorm_LBFGS = gradnorm_LBFGS[:min_len] + error_BFGS = error_BFGS[:min_len] + error_LBFGS = error_LBFGS[:min_len] + residual_BFGS = residual_BFGS[:min_len] + residual_LBFGS = residual_LBFGS[:min_len] + + iterations_interp = np.linspace(min(iterations), max(iterations), len(residual_BFGS)) + + gradnorm_BFGS = interp1d(iterations, gradnorm_BFGS, kind='cubic')(iterations_interp) + gradnorm_LBFGS = interp1d(iterations, gradnorm_LBFGS, kind='cubic')(iterations_interp) + error_BFGS = interp1d(iterations, error_BFGS, kind='cubic')(iterations_interp) + error_LBFGS = interp1d(iterations, error_LBFGS, kind='cubic')(iterations_interp) + residual_BFGS = interp1d(iterations, residual_BFGS, kind='cubic')(iterations_interp) + residual_LBFGS = interp1d(iterations, residual_LBFGS, kind='cubic')(iterations_interp) + + data = pd.DataFrame({ + 'Iterations': iterations_interp, + 'BFGS Gradient': gradnorm_BFGS, + 'LBFGS Gradient': gradnorm_LBFGS, + 'BFGS Error': error_BFGS, + 'LBFGS Error': error_LBFGS, + 'BFGS Residual': residual_BFGS, + 'LBFGS Residual': residual_LBFGS + }) + + data_long = pd.melt(data, id_vars=['Iterations'], var_name='algorithm', value_name='value') + + # Define custom color palette + custom_palette = { + 'BFGS Gradient': 'red', + 'BFGS Error': 'orange', + 'BFGS Residual': 'gold', + 'LBFGS Gradient': 'blue', + 'LBFGS Error': 'green', + 'LBFGS Residual': 'deepskyblue' + } + + # Create the plot + sns.lineplot(data=data_long, x='Iterations', y='value', hue='algorithm', palette=custom_palette) + + plt.yscale('log') + + handles, labels = plt.gca().get_legend_handles_labels() + labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0])) + plt.legend(handles, labels, loc='upper right', fontsize=15) + + conditioning = "LBFGS/" + folder + plt.savefig(PLOTS_PATH.format(conditioning, "BFGS-LBFGS-gradient-comparison.png")) + plt.show() + + +if __name__ == "__main__": + # plot all 3D plots + """ + plot_lambda_epsilon_3D('error') + plot_lambda_epsilon_3D('time') + plot_lambda_epsilon_3D('iterations') + plot_epsilon_memory_3D('error') + plot_epsilon_memory_3D('iterations') + """ + plot_norm_gradient_2D("well") + plot_norm_gradient_2D("ill") + # plot_awls_vs_exact_gradient_decrease_2D() + # plot_iteration_memory("well") + plot_iteration_memory("ill") + BFGS_comparison() diff --git a/project/plotting/QR.py b/project/plotting/QR.py new file mode 100644 index 0000000..656deab --- /dev/null +++ b/project/plotting/QR.py @@ -0,0 +1,68 @@ +from config import QR_RESULTS, PLOTS_PATH, nanoseconds_to_seconds, setup +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib.ticker as ticker +import seaborn as sns +import numpy as np + + +def plot_lambda(kind, smoothing_window=2): + if kind not in ['error', 'time']: + raise ValueError("kind must be one of 'error', 'time'") + + df = pd.read_csv(QR_RESULTS.format("statisticsQR-lambda-m300n20--{}.csv".format(kind))) + lambda_values = df['lambda'] + + if kind == 'time': + + setup(title='QR running time with respect to λ', y_label='Time (s)', x_label='λ') + meantime_values = df['meantime'] + stdtime_values = df['stdtime'] + sns.lineplot(data=df, x='lambda', y='meantime', label='Mean Time', color='blue') + plt.fill_between( + lambda_values, + meantime_values - stdtime_values, + meantime_values + stdtime_values, + alpha=0.3, + label='Standard Deviation', + color='lightblue' + ) + plt.ylim(0, 3.5 * max(meantime_values + stdtime_values)) + plt.gca().yaxis.set_major_formatter(ticker.FuncFormatter(nanoseconds_to_seconds)) + else: + setup(title='QR errors with respect to λ', y_label='Error', x_label='λ') + + df['relative'] = np.convolve(df['relative'], np.ones(smoothing_window) / smoothing_window, mode='same') + df['residual'] = np.convolve(df['residual'], np.ones(smoothing_window) / smoothing_window, mode='same') + df['stability'] = np.convolve(df['stability'], np.ones(smoothing_window) / smoothing_window, mode='same') + + sns.lineplot(data=df, x='lambda', y='relative', label='Relative Error', color='blue') + sns.lineplot(data=df, x='lambda', y='residual', label='Residual', color='red') + sns.lineplot(data=df, x='lambda', y='stability', label='Stability', color='green') + plt.yscale('log') + + plt.legend(loc='upper right', fontsize=15) + plt.xscale('log', base=10) + + plt.savefig(PLOTS_PATH.format('QR', "QR-lambda-{}.png".format(kind))) + plt.show() + + +def plot_forward_error(smoothing_window=5): + df = pd.read_csv(QR_RESULTS.format("statisticsQR-forward-m300n20--error.csv")) + setup(title='QR forward error with respect to λ', y_label='Error', x_label='λ') + df['forwardQ'] = np.convolve(df['forwardQ'], np.ones(smoothing_window) / smoothing_window, mode='same') + df['forwardR'] = np.convolve(df['forwardR'], np.ones(smoothing_window) / smoothing_window, mode='same') + sns.lineplot(data=df, x='lambda', y='forwardQ', label='Forward Error on Q', color='blue') + sns.lineplot(data=df, x='lambda', y='forwardR', label='Forward Error on R', color='red') + plt.yscale('log') + plt.xscale('log', base=10) + plt.legend(loc='upper right', fontsize=15) + plt.savefig(PLOTS_PATH.format('QR', "QR-forward_error.png")) + plt.show() + + +if __name__ == '__main__': + plot_lambda('error') + plot_lambda('time') + plot_forward_error() diff --git a/project/plotting/__init__.py b/project/plotting/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project/plotting/comparison.py b/project/plotting/comparison.py new file mode 100644 index 0000000..2291d72 --- /dev/null +++ b/project/plotting/comparison.py @@ -0,0 +1,271 @@ +import pandas as pd +from matplotlib import pyplot as plt, ticker +import seaborn as sns +from config import COMPARISON_RESULTS, PLOTS_PATH + + +def nanoseconds_to_seconds_2f(x, pos): + return f"{x / 10 ** 9:.2f}" + + +def nanoseconds_to_seconds_3f(x, pos): + return f"{x / 10 ** 9:.3f}" + + +def bytes_to_gib(x, pos): + return f"{x / 1024 ** 3:.2f}" + + +def QR_vs_LBFGS_time(conditioning, dimension="n"): + if conditioning not in ['ill', 'well']: + raise ValueError("conditioning must be one of 'ill', 'well'") + if dimension not in ['n', 'm']: + raise ValueError("dimension must be one of 'n', 'm'") + if dimension == 'n': + df = pd.read_csv(COMPARISON_RESULTS.format("QRvsLBFGS-n-m200-" + conditioning + "cond--time.csv")) + else: + df = pd.read_csv(COMPARISON_RESULTS.format("QRvsLBFGS-m-n50-" + conditioning + "cond--time.csv")) + sns.set(style="whitegrid") + sns.set_context('paper') + + fig, ax1 = plt.subplots(figsize=(11, 8)) + + sns.lineplot(data=df, x=dimension, y='meantimeQR', marker='o', label='Mean Time QR', color='blue', ax=ax1) + ax1.fill_between( + df[dimension], + df['meantimeQR'] - df['stdtimeQR'], + df['meantimeQR'] + df['stdtimeQR'], + alpha=0.3, + label='Standard Deviation QR', + color='lightblue' + ) + + sns.lineplot(data=df, x=dimension, y='meantimeLBFGS', marker='o', label='Mean Time L-BFGS', color='red', ax=ax1) + ax1.fill_between( + df[dimension], + df['meantimeLBFGS'] - df['stdtimeLBFGS'], + df['meantimeLBFGS'] + df['stdtimeLBFGS'], + alpha=0.3, + label='Standard Deviation L-BFGS', + color='lightsalmon' + ) + + ax1.set_title('QR vs LBFGS time and memory scalability on ' + conditioning + '-conditioned matrix', fontsize=20, + pad=20, + fontweight='bold') + ax1.set_xlabel(dimension, fontsize=15, labelpad=20) + ax1.set_ylabel('time (s)', fontsize=15, labelpad=20) + ax1.set_ylim(0, None) + + ax1.legend(loc='upper left', fontsize=15) + ax1.tick_params(axis='both', which='major', labelsize=15) + ax1.yaxis.set_major_formatter(ticker.FuncFormatter(nanoseconds_to_seconds_2f)) + + ax2 = ax1.twinx() + sns.lineplot(data=df, x=dimension, y='meanallocsQR', marker='o', label='GiB Allocation QR', color='green', ax=ax2) + sns.lineplot(data=df, x=dimension, y='meanallocsLBFGS', marker='o', label='GiB Allocation BFGS', color='purple', + ax=ax2) + ax2.set_ylabel('memory (GiB)', fontsize=15, labelpad=20) + ax2.set_ylim(0, None) + ax2.legend(loc='upper right', fontsize=15) + ax2.tick_params(axis='y', labelsize=15) + ax2.yaxis.set_major_formatter(ticker.FuncFormatter(bytes_to_gib)) + + plt.savefig(PLOTS_PATH.format("comparison", "QRvsLBFGS-scalability-time-" + conditioning + f"cond-{dimension}.png")) + plt.show() + + +def BFGSvsLBFGS_time(dimension="n"): + if dimension not in ['n', 'm']: + raise ValueError("dimension must be one of 'n', 'm'") + + if dimension == 'n': + df = pd.read_csv(COMPARISON_RESULTS.format("BFGSvsLBFGS-n-m200--time.csv")) + else: + df = pd.read_csv(COMPARISON_RESULTS.format("BFGSvsLBFGS-m-n50--time.csv")) + sns.set(style="whitegrid") + sns.set_context('paper') + + fig, ax1 = plt.subplots(figsize=(11, 8)) + + sns.lineplot(data=df, x=dimension, y='meantimeBFGS', marker='o', label='Mean Time BFGS', color='blue', ax=ax1) + ax1.fill_between( + df[dimension], + df['meantimeBFGS'] - df['stdtimeBFGS'], + df['meantimeBFGS'] + df['stdtimeBFGS'], + alpha=0.3, + label='Standard Deviation BFGS', + color='lightblue' + ) + + sns.lineplot(data=df, x=dimension, y='meantimeLBFGS', marker='o', label='Mean Time L-BFGS', color='red', ax=ax1) + ax1.fill_between( + df[dimension], + df['meantimeLBFGS'] - df['stdtimeLBFGS'], + df['meantimeLBFGS'] + df['stdtimeLBFGS'], + alpha=0.3, + label='Standard Deviation L-BFGS', + color='lightsalmon' + ) + + ax1.set_title('BFGS vs L-BFGS time and memory scalability on well-conditioned matrix', fontsize=20, pad=20, fontweight='bold') + ax1.set_xlabel(dimension, fontsize=15, labelpad=20) + ax1.set_ylabel('time (s)', fontsize=15, labelpad=20) + ax1.set_ylim(0, None) + + if dimension == 'n': + ax1.legend(loc='upper right', fontsize=15) + else: + ax1.legend(loc='upper left', fontsize=15) + ax1.tick_params(axis='both', which='major', labelsize=15) + ax1.yaxis.set_major_formatter(ticker.FuncFormatter(nanoseconds_to_seconds_3f)) + + ax2 = ax1.twinx() + sns.lineplot(data=df, x=dimension, y='meanallocsBFGS', marker='o', label='Bytes Allocation BFGS', color='green', + ax=ax2) + sns.lineplot(data=df, x=dimension, y='meanallocsLBFGS', marker='o', label='Bytes Allocation L-BFGS', color='purple', + ax=ax2) + ax2.set_ylabel('memory (B)', fontsize=15, labelpad=20) + ax2.set_ylim(0, None) + ax2.legend(loc='lower right', fontsize=15) + ax2.tick_params(axis='y', labelsize=15) + ax2.yaxis.set_major_formatter(ticker.FuncFormatter(bytes_to_gib)) + + plt.savefig(PLOTS_PATH.format("comparison", f"BFGSvsLBFGS-time-{dimension}.png")) + plt.show() + + +def quasi_newton_time(conditioning="well", dogleg=False): + df = pd.read_csv(COMPARISON_RESULTS.format(f"Quasi-Newton-Comparison-time-{conditioning}cond.csv")) + sns.set(style="whitegrid") + sns.set_context('paper') + + plt.figure(figsize=(11, 8)) + + sns.lineplot(data=df, x='m', y='meantimeLBFGS', marker='o', label='L-BFGS', color='blue') + plt.fill_between( + df['m'], + df['meantimeLBFGS'] - df['stdtimeLBFGS'], + df['meantimeLBFGS'] + df['stdtimeLBFGS'], + alpha=0.3, + label='Standard Deviation L-BFGS', + color='lightblue' + ) + + sns.lineplot(data=df, x='m', y='meantimeBFGS', marker='o', label='BFGS', color='red') + plt.fill_between( + df['m'], + df['meantimeBFGS'] - df['stdtimeBFGS'], + df['meantimeBFGS'] + df['stdtimeBFGS'], + alpha=0.3, + label='Standard Deviation BFGS', + color='lightsalmon' + ) + + sns.lineplot(data=df, x='m', y='meantimeDFP', marker='o', label='DFP', color='purple') + plt.fill_between( + df['m'], + df['meantimeDFP'] - df['stdtimeDFP'], + df['meantimeDFP'] + df['stdtimeDFP'], + alpha=0.3, + label='Standard Deviation DFP', + color='violet' + ) + + sns.lineplot(data=df, x='m', y='meantimeSR1', marker='o', label='SR1', color='violet') + plt.fill_between( + df['m'], + df['meantimeSR1'] - df['stdtimeSR1'], + df['meantimeSR1'] + df['stdtimeSR1'], + alpha=0.3, + label='Standard Deviation SR1', + color='lightsalmon' + ) + + if dogleg: + sns.lineplot(data=df, x='m', y='meantimeDFPDogleg', marker='o', label='DFP with Dogleg', color='orange') + plt.fill_between( + df['m'], + df['meantimeDFPDogleg'] - df['stdtimeDFPDogleg'], + df['meantimeDFPDogleg'] + df['stdtimeDFPDogleg'], + alpha=0.3, + label='Standard Deviation DFP with Dogleg', + color='lightsalmon' + ) + sns.lineplot(data=df, x='m', y='meantimeBFGSDogleg', marker='o', label='BFGS with Dogleg', color='green') + plt.fill_between( + df['m'], + df['meantimeBFGSDogleg'] - df['stdtimeBFGSDogleg'], + df['meantimeBFGSDogleg'] + df['stdtimeBFGSDogleg'], + alpha=0.3, + label='Standard Deviation BFGS with Dogleg', + color='lightgreen' + ) + + plt.title(f'Quasi-Newton methods running time on {conditioning}-conditioned matrix', fontsize=20, pad=20, + fontweight='bold') + plt.xlabel('m', fontsize=15, labelpad=20) + plt.ylabel('time (s)', fontsize=15, labelpad=20) + plt.ylim(0, None) + + handles, labels = plt.gca().get_legend_handles_labels() + + keyword = 'Standard Deviation' + + filtered_handles = [handle for label, handle in zip(labels, handles) if keyword not in label] + filtered_labels = [label for label in labels if keyword not in label] + + plt.legend(filtered_handles, filtered_labels, fontsize=15, loc='upper left') + plt.xticks(fontsize=15) + plt.yticks(fontsize=15) + + plt.gca().yaxis.set_major_formatter(ticker.FuncFormatter(nanoseconds_to_seconds_2f)) + + if dogleg: + plt.savefig(PLOTS_PATH.format("comparison", f"Quasi-Newton-Comparison-time-{conditioning}cond-Dogleg.png")) + else: + plt.savefig(PLOTS_PATH.format("comparison", f"Quasi-Newton-Comparison-time-{conditioning}cond.png")) + plt.show() + + +def quasi_newton_memory(conditioning="well"): + df = pd.read_csv(COMPARISON_RESULTS.format(f"Quasi-Newton-Comparison-time-{conditioning}cond.csv")) + sns.set(style="whitegrid") + sns.set_context('paper') + + plt.figure(figsize=(11, 8)) + + sns.lineplot(data=df, x='m', y='meanallocsLBFGS', marker='o', label='L-BFGS', color='blue') + sns.lineplot(data=df, x='m', y='meanallocsBFGS', marker='o', label='BFGS', color='red') + sns.lineplot(data=df, x='m', y='meanallocsBFGSDogleg', marker='o', label='BFGS with Dogleg', color='green') + sns.lineplot(data=df, x='m', y='meanallocsDFP', marker='o', label='DFP', color='purple') + sns.lineplot(data=df, x='m', y='meanallocsDFPDogleg', marker='o', label='DFP with Dogleg', color='orange') + sns.lineplot(data=df, x='m', y='meanallocsSR1', marker='o', label='SR1', color='violet') + + plt.title(f'Quasi-Newton methods memory allocation on {conditioning}-conditioned matrix', fontsize=20, pad=20, fontweight='bold') + plt.xlabel('m', fontsize=15, labelpad=20) + plt.ylabel('bytes (GiB)', fontsize=15, labelpad=20) + plt.ylim(-1, None) + + plt.legend(loc='upper left', fontsize=15) + plt.xticks(fontsize=15) + plt.yticks(fontsize=15) + + plt.gca().yaxis.set_major_formatter(ticker.FuncFormatter(bytes_to_gib)) + + plt.savefig(PLOTS_PATH.format("comparison", f"Quasi-Newton-Comparison-memory-{conditioning}cond.png")) + plt.show() + + +if __name__ == '__main__': + QR_vs_LBFGS_time("well", "m") + QR_vs_LBFGS_time("well", "n") + QR_vs_LBFGS_time("ill", "m") + QR_vs_LBFGS_time("ill", "n") + BFGSvsLBFGS_time("n") + BFGSvsLBFGS_time("m") + quasi_newton_time("ill") + quasi_newton_time("well", dogleg=True) + quasi_newton_time("well", dogleg=False) + quasi_newton_memory("ill") + quasi_newton_memory("well") diff --git a/project/plotting/conditioning.py b/project/plotting/conditioning.py new file mode 100644 index 0000000..8bdd9ce --- /dev/null +++ b/project/plotting/conditioning.py @@ -0,0 +1,32 @@ +# read a csv of two columns and plot them in 2d with seaborn + +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns +from config import CONDITIONING_RESULTS, PLOTS_PATH + + +def plot_conditioning(): + df = pd.read_csv(CONDITIONING_RESULTS.format('conditioning.csv')) + sns.set(style="whitegrid") + sns.set_context('paper') + + plt.figure(figsize=(11, 8)) + sns.lineplot(data=df, x='lambda', y='condnum', marker='o') + + plt.xticks(fontsize=15) + plt.yticks(fontsize=15) + + plt.xlabel('λ', fontsize=20) + plt.ylabel('k(X^)', fontsize=20) + plt.xscale('log') + plt.yscale('log') + + plt.title('Condition number of X^ with respect to λ', fontsize=25, pad=20, fontweight='bold') + + plt.savefig(PLOTS_PATH.format('conditioning', 'conditioning.png'), bbox_inches='tight', dpi=100) + plt.show() + + +if __name__ == '__main__': + plot_conditioning() diff --git a/project/plotting/config.py b/project/plotting/config.py new file mode 100644 index 0000000..5e71ee4 --- /dev/null +++ b/project/plotting/config.py @@ -0,0 +1,46 @@ +import os + +LBFGS_RESULTS = "/testing/results/LBFGS/{}/{}" +QR_RESULTS = "/testing/results/QR/{}" +CONDITIONING_RESULTS = "/testing/results/conditioning/{}" +COMPARISON_RESULTS = "/testing/results/comparison/{}" +PLOTS_PATH = "/testing/plots/{}/{}" + +# PROJECT FOLDER PATH +PROJECT_FOLDER_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +LBFGS_RESULTS = PROJECT_FOLDER_PATH + LBFGS_RESULTS +QR_RESULTS = PROJECT_FOLDER_PATH + QR_RESULTS +CONDITIONING_RESULTS = PROJECT_FOLDER_PATH + CONDITIONING_RESULTS +COMPARISON_RESULTS = PROJECT_FOLDER_PATH + COMPARISON_RESULTS +PLOTS_PATH = PROJECT_FOLDER_PATH + PLOTS_PATH + +# PLOTTING CONFIGURATIONS +from matplotlib import pyplot as plt +import seaborn as sns + + +def setup(title=None, x_label=None, y_label=None): + sns.set_style("whitegrid") + sns.set_context('paper') + plt.figure(figsize=(13, 9)) + plt.xticks(fontsize=15) + plt.yticks(fontsize=15) + plt.title(title, fontsize=20, pad=20, loc='center', fontweight='bold') + plt.xlabel(x_label, fontsize=15, labelpad=20) + plt.ylabel(y_label, fontsize=15, labelpad=20) + + +def nanoseconds_to_seconds(x, pos=None): + return f"{x / 10 ** 9:.4f}" + + +def log_tick_formatter(val, pos=None): + return f"$10^{{{int(val)}}}$" + + +def nanoseconds_to_milliseconds(x, pos=None): + return f"{x / 10 ** 6:.1f}" + +def scientific_notation(x, pos): + return '%1.1e' % x diff --git a/project/testing.ipynb b/project/testing.ipynb deleted file mode 100644 index 8476a6e..0000000 --- a/project/testing.ipynb +++ /dev/null @@ -1,3019 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "57a2b335-a7f3-4ace-a2d8-52219c4febc5", - "metadata": {}, - "outputs": [], - "source": [ - "include(\"thinQR.jl\")\n", - "using .thinQR" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "8b3d71f8-2f4f-44f0-8c97-dd9d78706163", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "x = [-1.5012955407352218e-16, 0.5000000000000001]\n" - ] - }, - { - "data": { - "text/plain": [ - "3-element Vector{Float64}:\n", - " 0.9999999999999991\n", - " 1.9999999999999996\n", - " 2.9999999999999996" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "A = [1. 2; 3. 4.; 5. 6.]\n", - "A = qrhous!(A)\n", - "b = [1., 2, 3]\n", - "x = A \\ b\n", - "@show x\n", - "A * x" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "afd34800-7b9a-4006-9849-071fa197d962", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "x = [5.810770900369532e-16, 0.4999999999999994]\n" - ] - }, - { - "data": { - "text/plain": [ - "3-element Vector{Float64}:\n", - " 0.9999999999999993\n", - " 1.9999999999999993\n", - " 2.999999999999999" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "using LinearAlgebra: \\, qr\n", - "A = [1. 2.; 3. 4.; 5. 6.]\n", - "(Q, R) = qr(A)\n", - "x = A \\ [1., 2, 3]\n", - "@show x\n", - "A * x" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "d7d24c53-5485-4f8a-bea3-3fe4ec1f456b", - "metadata": {}, - "outputs": [], - "source": [ - "using BenchmarkTools\n", - "A=randn(100, 10_000)\n", - "a=deepcopy(A)\n", - "y=randn(100);" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "652fe3c4-6b44-45e1-9bdc-5d7d41b55c6a", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "BenchmarkTools.Trial: 20 samples with 1 evaluation.\n", - " Range \u001b[90m(\u001b[39m\u001b[36m\u001b[1mmin\u001b[22m\u001b[39m … \u001b[35mmax\u001b[39m\u001b[90m): \u001b[39m\u001b[36m\u001b[1m246.781 ms\u001b[22m\u001b[39m … \u001b[35m278.597 ms\u001b[39m \u001b[90m┊\u001b[39m GC \u001b[90m(\u001b[39mmin … max\u001b[90m): \u001b[39m18.71% … 22.05%\n", - " Time \u001b[90m(\u001b[39m\u001b[34m\u001b[1mmedian\u001b[22m\u001b[39m\u001b[90m): \u001b[39m\u001b[34m\u001b[1m250.615 ms \u001b[22m\u001b[39m\u001b[90m┊\u001b[39m GC \u001b[90m(\u001b[39mmedian\u001b[90m): \u001b[39m19.94%\n", - " Time \u001b[90m(\u001b[39m\u001b[32m\u001b[1mmean\u001b[22m\u001b[39m ± \u001b[32mσ\u001b[39m\u001b[90m): \u001b[39m\u001b[32m\u001b[1m252.757 ms\u001b[22m\u001b[39m ± \u001b[32m 7.079 ms\u001b[39m \u001b[90m┊\u001b[39m GC \u001b[90m(\u001b[39mmean ± σ\u001b[90m): \u001b[39m20.18% ± 0.96%\n", - "\n", - " \u001b[39m \u001b[39m \u001b[39m \u001b[39m▄\u001b[39m▁\u001b[39m█\u001b[34m \u001b[39m\u001b[39m \u001b[39m \u001b[39m▁\u001b[39m▁\u001b[32m \u001b[39m\u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \n", - " \u001b[39m▆\u001b[39m▁\u001b[39m▁\u001b[39m█\u001b[39m█\u001b[39m█\u001b[34m▁\u001b[39m\u001b[39m▁\u001b[39m▆\u001b[39m█\u001b[39m█\u001b[32m▁\u001b[39m\u001b[39m▁\u001b[39m▁\u001b[39m▆\u001b[39m▆\u001b[39m▆\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▆\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▆\u001b[39m \u001b[39m▁\n", - " 247 ms\u001b[90m Histogram: frequency by time\u001b[39m 279 ms \u001b[0m\u001b[1m<\u001b[22m\n", - "\n", - " Memory estimate\u001b[90m: \u001b[39m\u001b[33m1.54 GiB\u001b[39m, allocs estimate\u001b[90m: \u001b[39m\u001b[33m4804\u001b[39m." - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "A = deepcopy(a)\n", - "@benchmark begin x = qrhous!($A) \\ $y; end" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "c8e93f5c-b32c-4334-bd97-9779360d5d81", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "BenchmarkTools.Trial: 85 samples with 1 evaluation.\n", - " Range \u001b[90m(\u001b[39m\u001b[36m\u001b[1mmin\u001b[22m\u001b[39m … \u001b[35mmax\u001b[39m\u001b[90m): \u001b[39m\u001b[36m\u001b[1m48.265 ms\u001b[22m\u001b[39m … \u001b[35m75.642 ms\u001b[39m \u001b[90m┊\u001b[39m GC \u001b[90m(\u001b[39mmin … max\u001b[90m): \u001b[39m0.00% … 6.42%\n", - " Time \u001b[90m(\u001b[39m\u001b[34m\u001b[1mmedian\u001b[22m\u001b[39m\u001b[90m): \u001b[39m\u001b[34m\u001b[1m60.888 ms \u001b[22m\u001b[39m\u001b[90m┊\u001b[39m GC \u001b[90m(\u001b[39mmedian\u001b[90m): \u001b[39m0.00%\n", - " Time \u001b[90m(\u001b[39m\u001b[32m\u001b[1mmean\u001b[22m\u001b[39m ± \u001b[32mσ\u001b[39m\u001b[90m): \u001b[39m\u001b[32m\u001b[1m59.241 ms\u001b[22m\u001b[39m ± \u001b[32m 7.561 ms\u001b[39m \u001b[90m┊\u001b[39m GC \u001b[90m(\u001b[39mmean ± σ\u001b[90m): \u001b[39m0.76% ± 1.71%\n", - "\n", - " \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m▂\u001b[39m█\u001b[39m█\u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[32m \u001b[39m\u001b[39m \u001b[39m \u001b[39m \u001b[34m▂\u001b[39m\u001b[39m▅\u001b[39m▂\u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \u001b[39m \n", - " \u001b[39m▅\u001b[39m▄\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m█\u001b[39m█\u001b[39m█\u001b[39m▇\u001b[39m▄\u001b[39m▁\u001b[39m▅\u001b[39m▁\u001b[39m▅\u001b[39m▁\u001b[39m▁\u001b[39m▄\u001b[39m▄\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▄\u001b[39m▅\u001b[39m▁\u001b[32m▁\u001b[39m\u001b[39m▁\u001b[39m▄\u001b[39m▄\u001b[34m█\u001b[39m\u001b[39m█\u001b[39m█\u001b[39m▅\u001b[39m▄\u001b[39m▄\u001b[39m▁\u001b[39m▅\u001b[39m▅\u001b[39m▅\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▁\u001b[39m▄\u001b[39m▅\u001b[39m▁\u001b[39m▄\u001b[39m▅\u001b[39m▄\u001b[39m▁\u001b[39m▄\u001b[39m▄\u001b[39m▅\u001b[39m▅\u001b[39m▁\u001b[39m▄\u001b[39m▄\u001b[39m▄\u001b[39m \u001b[39m▁\n", - " 48.3 ms\u001b[90m Histogram: frequency by time\u001b[39m 73.7 ms \u001b[0m\u001b[1m<\u001b[22m\n", - "\n", - " Memory estimate\u001b[90m: \u001b[39m\u001b[33m7.92 MiB\u001b[39m, allocs estimate\u001b[90m: \u001b[39m\u001b[33m624\u001b[39m." - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "using LinearAlgebra\n", - "\n", - "@benchmark begin x = $a \\ $y; end" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "43d521a7-6bf4-45c6-abb6-0a07a666570d", - "metadata": {}, - "outputs": [], - "source": [ - "using Profile, ProfileSVG" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "9443e1c4-03dd-46cb-a8ea-652c6862dbcf", - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "\n", - "Profile results\n", - "\n", - "\n", - " in :-1\n", - "\n", - "#15 in task.jl:514\n", - "\n", - "eventloop in eventloop.jl:8\n", - "\n", - "invokelatest in essentials.jl:816\n", - "\n", - "#invokelatest#2 in essentials.jl:819\n", - "\n", - "execute_request in execute_request.jl:67\n", - "\n", - "softscope_include_string in SoftGlobalScope.jl:65\n", - "\n", - "include_string in loading.jl:1903\n", - "\n", - "eval in boot.jl:370\n", - "\n", - "\\ in thinQR.jl:108\n", - "\n", - "qyhoust in thinQR.jl:57\n", - "\n", - "getindex in abstractarray.jl:1296\n", - "\n", - "_getindex in multidimensional.jl:861\n", - "\n", - "_unsafe_getindex in multidimensional.jl:875\n", - "\n", - "_unsafe_getindex! in multidimensional.jl:884\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:889\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "- in arraymath.jl:8\n", - "\n", - "broadcast_preserving_zero_d in broadcast.jl:862\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "qrhous! in thinQR.jl:22\n", - "\n", - "getindex in abstractarray.jl:1296\n", - "\n", - "_getindex in multidimensional.jl:861\n", - "\n", - "_unsafe_getindex in multidimensional.jl:875\n", - "\n", - "_unsafe_getindex! in multidimensional.jl:884\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:889\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "qrhous! in thinQR.jl:31\n", - "\n", - "getindex in abstractarray.jl:1296\n", - "\n", - "_getindex in multidimensional.jl:861\n", - "\n", - "_unsafe_getindex in multidimensional.jl:873\n", - "\n", - "similar in abstractarray.jl:838\n", - "\n", - "similar in array.jl:374\n", - "\n", - "Array in boot.jl:487\n", - "\n", - "Array in boot.jl:479\n", - "\n", - "_unsafe_getindex in multidimensional.jl:875\n", - "\n", - "_unsafe_getindex! in multidimensional.jl:884\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:889\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "_unsafe_getindex in multidimensional.jl:875\n", - "\n", - "_unsafe_getindex! in multidimensional.jl:884\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:887\n", - "\n", - "macro expansion in multidimensional.jl:889\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "setindex! in abstractarray.jl:1399\n", - "\n", - "_setindex! in multidimensional.jl:916\n", - "\n", - "_unsafe_setindex! in multidimensional.jl:927\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:930\n", - "\n", - "macro expansion in multidimensional.jl:932\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "macro expansion in multidimensional.jl:933\n", - "\n", - "iterate in array.jl:893\n", - "\n", - "getindex in essentials.jl:13\n", - "\n", - "- in int.jl:1042\n", - "\n", - "- in int.jl:86\n", - "\n", - "&lt; in int.jl:494\n", - "\n", - "&lt; in int.jl:487\n", - "\n", - "macro expansion in cartesian.jl:66\n", - "\n", - "iterate in range.jl:891\n", - "\n", - "== in promotion.jl:499\n", - "\n", - "- in arraymath.jl:8\n", - "\n", - "broadcast_preserving_zero_d in broadcast.jl:862\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "getindex in broadcast.jl:610\n", - "\n", - "_broadcast_getindex in broadcast.jl:655\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_broadcast_getindex in broadcast.jl:649\n", - "\n", - "getindex in multidimensional.jl:668\n", - "\n", - "getindex in essentials.jl:14\n", - "\n", - "setindex! in multidimensional.jl:670\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "macro expansion in simdloop.jl:78\n", - "\n", - "+ in int.jl:87\n", - "\n", - "similar in broadcast.jl:211\n", - "\n", - "similar in broadcast.jl:212\n", - "\n", - "similar in abstractarray.jl:883\n", - "\n", - "similar in abstractarray.jl:884\n", - "\n", - "Array in boot.jl:494\n", - "\n", - "Array in boot.jl:487\n", - "\n", - "Array in boot.jl:479\n", - "\n", - "* in adjtrans.jl:439\n", - "\n", - "broadcast in broadcast.jl:811\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "getindex in broadcast.jl:610\n", - "\n", - "_broadcast_getindex in broadcast.jl:655\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_broadcast_getindex in broadcast.jl:649\n", - "\n", - "getindex in multidimensional.jl:668\n", - "\n", - "getindex in essentials.jl:13\n", - "\n", - "setindex! in multidimensional.jl:670\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "macro expansion in simdloop.jl:78\n", - "\n", - "+ in int.jl:87\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ], - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "\n", - "Profile results\n", - "\n", - "\n", - " in :-1\n", - "\n", - "#15 in task.jl:514\n", - "\n", - "eventloop in eventloop.jl:8\n", - "\n", - "invokelatest in essentials.jl:816\n", - "\n", - "#invokelatest#2 in essentials.jl:819\n", - "\n", - "execute_request in execute_request.jl:67\n", - "\n", - "softscope_include_string in SoftGlobalScope.jl:65\n", - "\n", - "include_string in loading.jl:1903\n", - "\n", - "eval in boot.jl:370\n", - "\n", - "\\ in thinQR.jl:108\n", - "\n", - "qyhoust in thinQR.jl:57\n", - "\n", - "getindex in abstractarray.jl:1296\n", - "\n", - "_getindex in multidimensional.jl:861\n", - "\n", - "_unsafe_getindex in multidimensional.jl:875\n", - "\n", - "_unsafe_getindex! in multidimensional.jl:884\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:889\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "- in arraymath.jl:8\n", - "\n", - "broadcast_preserving_zero_d in broadcast.jl:862\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "qrhous! in thinQR.jl:22\n", - "\n", - "getindex in abstractarray.jl:1296\n", - "\n", - "_getindex in multidimensional.jl:861\n", - "\n", - "_unsafe_getindex in multidimensional.jl:875\n", - "\n", - "_unsafe_getindex! in multidimensional.jl:884\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:889\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "qrhous! in thinQR.jl:31\n", - "\n", - "getindex in abstractarray.jl:1296\n", - "\n", - "_getindex in multidimensional.jl:861\n", - "\n", - "_unsafe_getindex in multidimensional.jl:873\n", - "\n", - "similar in abstractarray.jl:838\n", - "\n", - "similar in array.jl:374\n", - "\n", - "Array in boot.jl:487\n", - "\n", - "Array in boot.jl:479\n", - "\n", - "_unsafe_getindex in multidimensional.jl:875\n", - "\n", - "_unsafe_getindex! in multidimensional.jl:884\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:889\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "_unsafe_getindex in multidimensional.jl:875\n", - "\n", - "_unsafe_getindex! in multidimensional.jl:884\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:887\n", - "\n", - "macro expansion in multidimensional.jl:889\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "setindex! in abstractarray.jl:1399\n", - "\n", - "_setindex! in multidimensional.jl:916\n", - "\n", - "_unsafe_setindex! in multidimensional.jl:927\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:930\n", - "\n", - "macro expansion in multidimensional.jl:932\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "macro expansion in multidimensional.jl:933\n", - "\n", - "iterate in array.jl:893\n", - "\n", - "getindex in essentials.jl:13\n", - "\n", - "- in int.jl:1042\n", - "\n", - "- in int.jl:86\n", - "\n", - "&lt; in int.jl:494\n", - "\n", - "&lt; in int.jl:487\n", - "\n", - "macro expansion in cartesian.jl:66\n", - "\n", - "iterate in range.jl:891\n", - "\n", - "== in promotion.jl:499\n", - "\n", - "- in arraymath.jl:8\n", - "\n", - "broadcast_preserving_zero_d in broadcast.jl:862\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "getindex in broadcast.jl:610\n", - "\n", - "_broadcast_getindex in broadcast.jl:655\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_broadcast_getindex in broadcast.jl:649\n", - "\n", - "getindex in multidimensional.jl:668\n", - "\n", - "getindex in essentials.jl:14\n", - "\n", - "setindex! in multidimensional.jl:670\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "macro expansion in simdloop.jl:78\n", - "\n", - "+ in int.jl:87\n", - "\n", - "similar in broadcast.jl:211\n", - "\n", - "similar in broadcast.jl:212\n", - "\n", - "similar in abstractarray.jl:883\n", - "\n", - "similar in abstractarray.jl:884\n", - "\n", - "Array in boot.jl:494\n", - "\n", - "Array in boot.jl:487\n", - "\n", - "Array in boot.jl:479\n", - "\n", - "* in adjtrans.jl:439\n", - "\n", - "broadcast in broadcast.jl:811\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "getindex in broadcast.jl:610\n", - "\n", - "_broadcast_getindex in broadcast.jl:655\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_broadcast_getindex in broadcast.jl:649\n", - "\n", - "getindex in multidimensional.jl:668\n", - "\n", - "getindex in essentials.jl:13\n", - "\n", - "setindex! in multidimensional.jl:670\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "macro expansion in simdloop.jl:78\n", - "\n", - "+ in int.jl:87\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ], - "text/plain": [ - "ProfileSVG.FGConfig(Node(FlameGraphs.NodeData(ip:0x0, 0x01, 1:212)), Dict{Symbol, Any}(), FlameGraphs.FlameColors(ColorTypes.RGB{FixedPointNumbers.N0f8}[RGB{N0f8}(0.882,0.698,1.0), RGB{N0f8}(0.435,0.863,0.569), RGB{N0f8}(0.0,0.71,0.545), RGB{N0f8}(0.173,0.639,1.0)], RGB{N0f8}(1.0,1.0,1.0), RGB{N0f8}(0.0,0.0,0.0), ColorTypes.RGB{FixedPointNumbers.N0f8}[RGB{N0f8}(0.953,0.0,0.302), RGB{N0f8}(0.894,0.0,0.255), RGB{N0f8}(0.831,0.129,0.216), RGB{N0f8}(0.773,0.192,0.184)], ColorTypes.RGB{FixedPointNumbers.N0f8}[RGB{N0f8}(1.0,0.627,0.0), RGB{N0f8}(1.0,0.643,0.0), RGB{N0f8}(0.965,0.651,0.039), RGB{N0f8}(0.894,0.655,0.11)]), :fcolor, :fcolor, 1.0, false, 50, 2000, 960.0, 0.0, 2.0, \"inherit\", 12.0, false, :none, 0.001)" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "A = a\n", - "@profview qrhous!(A) \\ y" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "fb2484bd-fd33-4b42-bc14-1006c4d9811d", - "metadata": {}, - "outputs": [], - "source": [ - "include(\"housQR.jl\")\n", - "using .housQR" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "eb827167-7fa3-4106-9bac-71a3c237ad5b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "x = [1.3333333333333313, -0.3333333333333316]\n" - ] - }, - { - "data": { - "text/plain": [ - "3-element Vector{Float64}:\n", - " 0.6666666666666674\n", - " 2.6666666666666674\n", - " 4.666666666666667" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "AAA = [1. 2; 3 4; 5 6]\n", - "q_r = qrfact(AAA)\n", - "\n", - "b = [1., 2, 5]\n", - "x = q_r \\ b\n", - "@show x\n", - "q_r.Q * q_r.R * x" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "41d9dca0-6880-4133-8dc3-c35d5c0c9a49", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "size(v) = (100,)\n", - "size(x) = (10000,)\n", - "size(A.A) = (100, 10000)\n", - "size(A.d) = (10000,)\n", - "m = 10000\n" - ] - }, - { - "ename": "LoadError", - "evalue": "BoundsError: attempt to access 100-element Vector{Float64} at index [10000]", - "output_type": "error", - "traceback": [ - "BoundsError: attempt to access 100-element Vector{Float64} at index [10000]", - "", - "Stacktrace:", - " [1] getindex", - " @ ./essentials.jl:13 [inlined]", - " [2] maybeview(A::Vector{Float64}, args::Int64)", - " @ Base ./views.jl:149", - " [3] \\(A::QRhous{Float64}, b::Vector{Float64})", - " @ Main.housQR ~/Documents/University/Corsi/Computational Mathematics for Learning and Data Analysis/CMLDA/project/housQR.jl:125", - " [4] top-level scope", - " @ ./timing.jl:273 [inlined]", - " [5] top-level scope", - " @ ./In[3]:0" - ] - } - ], - "source": [ - "asd = qrfact(A);\n", - "\n", - "@time asd \\ y;" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "ee126d8a-ef8e-4cd9-b63c-6fb8696e9766", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 0.005245 seconds (2.92 k allocations: 22.947 MiB, 46.95% gc time)\n" - ] - } - ], - "source": [ - "asd = qrfact(A);\n", - "\n", - "@time asd \\ y;" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "13fd94ac-339b-4a9b-8532-10b4b05860bb", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "\n", - "Profile results\n", - "\n", - "\n", - " in :-1\n", - "\n", - "#15 in task.jl:514\n", - "\n", - "eventloop in eventloop.jl:8\n", - "\n", - "invokelatest in essentials.jl:816\n", - "\n", - "#invokelatest#2 in essentials.jl:819\n", - "\n", - "execute_request in execute_request.jl:67\n", - "\n", - "softscope_include_string in SoftGlobalScope.jl:65\n", - "\n", - "include_string in loading.jl:1903\n", - "\n", - "eval in boot.jl:370\n", - "\n", - "\\ in housQR.jl:114\n", - "\n", - "qyhoust in housQR.jl:62\n", - "\n", - "getindex in abstractarray.jl:1296\n", - "\n", - "_getindex in multidimensional.jl:861\n", - "\n", - "_unsafe_getindex in multidimensional.jl:875\n", - "\n", - "_unsafe_getindex! in multidimensional.jl:884\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:889\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "- in arraymath.jl:8\n", - "\n", - "broadcast_preserving_zero_d in broadcast.jl:862\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "getindex in broadcast.jl:610\n", - "\n", - "_broadcast_getindex in broadcast.jl:655\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_broadcast_getindex in broadcast.jl:649\n", - "\n", - "getindex in essentials.jl:13\n", - "\n", - "qrfact in housQR.jl:40\n", - "\n", - "setindex! in abstractarray.jl:1399\n", - "\n", - "_setindex! in multidimensional.jl:916\n", - "\n", - "_unsafe_setindex! in multidimensional.jl:927\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:930\n", - "\n", - "macro expansion in multidimensional.jl:932\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "macro expansion in multidimensional.jl:933\n", - "\n", - "iterate in array.jl:893\n", - "\n", - "macro expansion in cartesian.jl:66\n", - "\n", - "iterate in range.jl:891\n", - "\n", - "== in promotion.jl:499\n", - "\n", - "- in arraymath.jl:8\n", - "\n", - "broadcast_preserving_zero_d in broadcast.jl:862\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "getindex in broadcast.jl:610\n", - "\n", - "_broadcast_getindex in broadcast.jl:655\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_broadcast_getindex in broadcast.jl:649\n", - "\n", - "getindex in abstractarray.jl:1296\n", - "\n", - "_getindex in abstractarray.jl:1346\n", - "\n", - "getindex in subarray.jl:286\n", - "\n", - "getindex in essentials.jl:14\n", - "\n", - "setindex! in multidimensional.jl:670\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "similar in broadcast.jl:211\n", - "\n", - "similar in broadcast.jl:212\n", - "\n", - "similar in abstractarray.jl:883\n", - "\n", - "similar in abstractarray.jl:884\n", - "\n", - "Array in boot.jl:494\n", - "\n", - "Array in boot.jl:487\n", - "\n", - "Array in boot.jl:479\n", - "\n", - "* in matmul.jl:1162\n", - "\n", - "broadcast in broadcast.jl:811\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "getindex in broadcast.jl:610\n", - "\n", - "_broadcast_getindex in broadcast.jl:655\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_broadcast_getindex in broadcast.jl:649\n", - "\n", - "getindex in multidimensional.jl:668\n", - "\n", - "getindex in essentials.jl:13\n", - "\n", - "setindex! in multidimensional.jl:670\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "similar in broadcast.jl:211\n", - "\n", - "similar in broadcast.jl:212\n", - "\n", - "similar in abstractarray.jl:883\n", - "\n", - "similar in abstractarray.jl:884\n", - "\n", - "Array in boot.jl:494\n", - "\n", - "Array in boot.jl:487\n", - "\n", - "Array in boot.jl:479\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ], - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "\n", - "Profile results\n", - "\n", - "\n", - " in :-1\n", - "\n", - "#15 in task.jl:514\n", - "\n", - "eventloop in eventloop.jl:8\n", - "\n", - "invokelatest in essentials.jl:816\n", - "\n", - "#invokelatest#2 in essentials.jl:819\n", - "\n", - "execute_request in execute_request.jl:67\n", - "\n", - "softscope_include_string in SoftGlobalScope.jl:65\n", - "\n", - "include_string in loading.jl:1903\n", - "\n", - "eval in boot.jl:370\n", - "\n", - "\\ in housQR.jl:114\n", - "\n", - "qyhoust in housQR.jl:62\n", - "\n", - "getindex in abstractarray.jl:1296\n", - "\n", - "_getindex in multidimensional.jl:861\n", - "\n", - "_unsafe_getindex in multidimensional.jl:875\n", - "\n", - "_unsafe_getindex! in multidimensional.jl:884\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:889\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "- in arraymath.jl:8\n", - "\n", - "broadcast_preserving_zero_d in broadcast.jl:862\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "setindex! in array.jl:969\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "getindex in broadcast.jl:610\n", - "\n", - "_broadcast_getindex in broadcast.jl:655\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_broadcast_getindex in broadcast.jl:649\n", - "\n", - "getindex in essentials.jl:13\n", - "\n", - "qrfact in housQR.jl:40\n", - "\n", - "setindex! in abstractarray.jl:1399\n", - "\n", - "_setindex! in multidimensional.jl:916\n", - "\n", - "_unsafe_setindex! in multidimensional.jl:927\n", - "\n", - "macro expansion in cartesian.jl:64\n", - "\n", - "macro expansion in multidimensional.jl:930\n", - "\n", - "macro expansion in multidimensional.jl:932\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "macro expansion in multidimensional.jl:933\n", - "\n", - "iterate in array.jl:893\n", - "\n", - "macro expansion in cartesian.jl:66\n", - "\n", - "iterate in range.jl:891\n", - "\n", - "== in promotion.jl:499\n", - "\n", - "- in arraymath.jl:8\n", - "\n", - "broadcast_preserving_zero_d in broadcast.jl:862\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "getindex in broadcast.jl:610\n", - "\n", - "_broadcast_getindex in broadcast.jl:655\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_broadcast_getindex in broadcast.jl:649\n", - "\n", - "getindex in abstractarray.jl:1296\n", - "\n", - "_getindex in abstractarray.jl:1346\n", - "\n", - "getindex in subarray.jl:286\n", - "\n", - "getindex in essentials.jl:14\n", - "\n", - "setindex! in multidimensional.jl:670\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "similar in broadcast.jl:211\n", - "\n", - "similar in broadcast.jl:212\n", - "\n", - "similar in abstractarray.jl:883\n", - "\n", - "similar in abstractarray.jl:884\n", - "\n", - "Array in boot.jl:494\n", - "\n", - "Array in boot.jl:487\n", - "\n", - "Array in boot.jl:479\n", - "\n", - "* in matmul.jl:1162\n", - "\n", - "broadcast in broadcast.jl:811\n", - "\n", - "materialize in broadcast.jl:873\n", - "\n", - "copy in broadcast.jl:898\n", - "\n", - "copyto! in broadcast.jl:926\n", - "\n", - "copyto! in broadcast.jl:973\n", - "\n", - "macro expansion in simdloop.jl:77\n", - "\n", - "macro expansion in broadcast.jl:974\n", - "\n", - "getindex in broadcast.jl:610\n", - "\n", - "_broadcast_getindex in broadcast.jl:655\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_getindex in broadcast.jl:679\n", - "\n", - "_broadcast_getindex in broadcast.jl:649\n", - "\n", - "getindex in multidimensional.jl:668\n", - "\n", - "getindex in essentials.jl:13\n", - "\n", - "setindex! in multidimensional.jl:670\n", - "\n", - "setindex! in array.jl:971\n", - "\n", - "similar in broadcast.jl:211\n", - "\n", - "similar in broadcast.jl:212\n", - "\n", - "similar in abstractarray.jl:883\n", - "\n", - "similar in abstractarray.jl:884\n", - "\n", - "Array in boot.jl:494\n", - "\n", - "Array in boot.jl:487\n", - "\n", - "Array in boot.jl:479\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ], - "text/plain": [ - "ProfileSVG.FGConfig(Node(FlameGraphs.NodeData(ip:0x0, 0x01, 1:122)), Dict{Symbol, Any}(), FlameGraphs.FlameColors(ColorTypes.RGB{FixedPointNumbers.N0f8}[RGB{N0f8}(0.882,0.698,1.0), RGB{N0f8}(0.435,0.863,0.569), RGB{N0f8}(0.0,0.71,0.545), RGB{N0f8}(0.173,0.639,1.0)], RGB{N0f8}(1.0,1.0,1.0), RGB{N0f8}(0.0,0.0,0.0), ColorTypes.RGB{FixedPointNumbers.N0f8}[RGB{N0f8}(0.953,0.0,0.302), RGB{N0f8}(0.894,0.0,0.255), RGB{N0f8}(0.831,0.129,0.216), RGB{N0f8}(0.773,0.192,0.184)], ColorTypes.RGB{FixedPointNumbers.N0f8}[RGB{N0f8}(1.0,0.627,0.0), RGB{N0f8}(1.0,0.643,0.0), RGB{N0f8}(0.965,0.651,0.039), RGB{N0f8}(0.894,0.655,0.11)]), :fcolor, :fcolor, 1.0, false, 50, 2000, 960.0, 0.0, 2.0, \"inherit\", 12.0, false, :none, 0.001)" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "@profview qrfact(A) \\ y" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 1.9.4", - "language": "julia", - "name": "julia-1.9" - }, - "language_info": { - "file_extension": ".jl", - "mimetype": "application/julia", - "name": "julia", - "version": "1.9.4" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/project/testing/L-BFGS/BFGS.ipynb b/project/testing/L-BFGS/BFGS.ipynb new file mode 100644 index 0000000..1fa12e8 --- /dev/null +++ b/project/testing/L-BFGS/BFGS.ipynb @@ -0,0 +1,176 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "include(\"../../L-BFGS/OracleFunction.jl\")\n", + "include(\"../../L-BFGS/BFGS.jl\")\n", + "include(\"../../utilities/genFunc.jl\")\n", + "using .BFGS\n", + "using .OracleFunction\n", + "using LinearAlgebra, BenchmarkTools, CSV, DataFrames\n", + "\n", + "baseDir = joinpath(\"../\", \"results/LBFGS/comparison_BFGS/\")\n", + "mkpath(baseDir);" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LeastSquaresF{Float64, Main.OracleFunction.var\"#f#1\"{Matrix{Float64}, Vector{Float64}}, Main.OracleFunction.var\"#df#2\"{Matrix{Float64}, Vector{Float64}}}(OracleF{Float64, Main.OracleFunction.var\"#f#1\"{Matrix{Float64}, Vector{Float64}}, Main.OracleFunction.var\"#df#2\"{Matrix{Float64}, Vector{Float64}}}([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 … 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], Main.OracleFunction.var\"#f#1\"{Matrix{Float64}, Vector{Float64}}([0.6269851049934334 -0.9953855131718308 … 0.26819190323331954 -0.6780465661809252; -0.9332327939739362 0.8084805224537295 … 0.28452273460460176 0.9029306010736042; … ; 0.0 0.0 … 10.0 0.0; 0.0 0.0 … 0.0 10.0], [-10.186364190714537, -1.6231132115458327, 15.375526057866429, 1.5006156140417093, -14.546392908739245, -4.939853634188801, -0.2053418568399169, 20.47017872706546, 1.0276498922046493, -12.900436004701787 … 3.3479087325325874, 4.529977532247315, 9.395681468606902, -9.098074604739619, -8.545976276298948, 5.1912832757085265, -8.133605566489734, -7.81791180224994, 8.53647733342897, 0.2170916132884937]), Main.OracleFunction.var\"#df#2\"{Matrix{Float64}, Vector{Float64}}([0.6269851049934334 -0.9953855131718308 … 0.26819190323331954 -0.6780465661809252; -0.9332327939739362 0.8084805224537295 … 0.28452273460460176 0.9029306010736042; … ; 0.0 0.0 … 10.0 0.0; 0.0 0.0 … 0.0 10.0], [-10.186364190714537, -1.6231132115458327, 15.375526057866429, 1.5006156140417093, -14.546392908739245, -4.939853634188801, -0.2053418568399169, 20.47017872706546, 1.0276498922046493, -12.900436004701787 … 3.3479087325325874, 4.529977532247315, 9.395681468606902, -9.098074604739619, -8.545976276298948, 5.1912832757085265, -8.133605566489734, -7.81791180224994, 8.53647733342897, 0.2170916132884937])), [0.6269851049934334 -0.9953855131718308 … 0.26819190323331954 -0.6780465661809252; -0.9332327939739362 0.8084805224537295 … 0.28452273460460176 0.9029306010736042; … ; 0.0 0.0 … 10.0 0.0; 0.0 0.0 … 0.0 10.0], [-10.186364190714537, -1.6231132115458327, 15.375526057866429, 1.5006156140417093, -14.546392908739245, -4.939853634188801, -0.2053418568399169, 20.47017872706546, 1.0276498922046493, -12.900436004701787 … 3.3479087325325874, 4.529977532247315, 9.395681468606902, -9.098074604739619, -8.545976276298948, 5.1912832757085265, -8.133605566489734, -7.81791180224994, 8.53647733342897, 0.2170916132884937], [106.26650786511657 -1.0664727828949865 … 2.4445684049045426 0.0322174082238121; -1.0664727828949865 106.1318558809136 … -0.6525752258209305 1.2988114776625301; … ; 2.4445684049045426 -0.6525752258209305 … 106.71985607542891 0.16719323022752428; 0.0322174082238121 1.2988114776625301 … 0.16719323022752428 105.42561554395674], [2.0243350631400165 57.59287736844317 … 68.90017274988415 -7.460256118037204])" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# parameters for exact LS vs AWLS vs quadratic function \n", + "λ = 10^1\n", + "ϵ = 10^-14\n", + "maxIters = 1:200\n", + "m = 1000\n", + "n = 20\n", + "num_trials = 10\n", + "\n", + "gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + "non_quadratic = OracleF(ones(size(gf[:X_hat], 2)),\n", + " (x) -> norm(gf[:X_hat] * x - gf[:y_hat]),\n", + " (x) -> inv(norm(gf[:X_hat] * x - gf[:y_hat])) * gf[:X_hat]' * (gf[:X_hat] * x - gf[:y_hat])\n", + " )\n", + "quadratic = OracleF(ones(size(gf[:X_hat], 2)),\n", + " (x) -> norm(gf[:X_hat] * x - gf[:y_hat]) ^ 2,\n", + " (x) -> 2 * gf[:X_hat]' * (gf[:X_hat] * x - gf[:y_hat])\n", + " )\n", + "ls = LeastSquaresF(gf)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done trial 1\n", + "Done trial 2\n", + "Done trial 3\n", + "Done trial 4\n", + "Done trial 5\n", + "Done trial 6\n", + "Done trial 7\n", + "Done trial 8\n", + "Done trial 9\n", + "Done trial 10\n" + ] + }, + { + "data": { + "text/plain": [ + "\"../results/LBFGS/comparison_BFGS/statisticsBFGS-iterations-m1000n20--error-norm.csv\"" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "### residual, relative error and norm of gradient with respect to iterations with exact line search\n", + "using Statistics, CSV\n", + "\n", + "# Preallocate arrays\n", + "num_iterations = length(maxIters)\n", + "\n", + "gradients = zeros(num_trials, num_iterations)\n", + "residuals = zeros(num_trials, num_iterations)\n", + "relative_errors = zeros(num_trials, num_iterations)\n", + "\n", + "for trial in 1:num_trials\n", + "\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " for (i, maxIter) in enumerate(maxIters)\n", + " t = BroydenFletcherGoldfarbShanno(ls, ϵ=ϵ, MaxEvaluations=maxIter)\n", + "\n", + " relative_errors[trial, i] = norm(t[:x] - gf[:w_star]) / norm(gf[:w_star])\n", + " residuals[trial, i] = norm(gf[:X_hat] * t[:x] - gf[:y_hat]) / norm(gf[:y_hat])\n", + " gradients[trial, i] = norm(t[:grad])\n", + " end\n", + "\n", + " println(\"Done trial \", trial)\n", + " \n", + "end\n", + "\n", + "# Calculate mean and standard deviation\n", + "mean_relative = mean(relative_errors, dims=1)'\n", + "std_relative = std(relative_errors, dims=1)'\n", + "mean_residual = mean(residuals, dims=1)'\n", + "std_residual = std(residuals, dims=1)'\n", + "mean_gradient = mean(gradients, dims=1)'\n", + "std_gradient = std(gradients, dims=1)'\n", + "\n", + "\n", + "# Write results to CSV\n", + "outputvsc = joinpath(baseDir, \"statisticsBFGS-iterations-m\" * string(m) * \"n\" * string(n) * \"--error-norm.csv\");\n", + "\n", + "accData = Dict(\n", + " :maxiterations => Array{Int64}(undef, 0),\n", + " :mean_relative => Array{Float64}(undef, 0),\n", + " :std_relative => Array{Float64}(undef, 0),\n", + " :mean_residual => Array{Float64}(undef, 0),\n", + " :std_residual => Array{Float64}(undef, 0),\n", + " :mean_gradient => Array{Float64}(undef, 0),\n", + " :std_gradient => Array{Float64}(undef, 0)\n", + ")\n", + "\n", + "# create dataframe with columns from arrays\n", + "for maxIter ∈ maxIters\n", + " push!(accData[:maxiterations], maxIter)\n", + " push!(accData[:mean_relative], mean_relative[maxIter])\n", + " push!(accData[:std_relative], std_relative[maxIter])\n", + " push!(accData[:mean_residual], mean_residual[maxIter])\n", + " push!(accData[:std_residual], std_residual[maxIter])\n", + " push!(accData[:mean_gradient], mean_gradient[maxIter])\n", + " push!(accData[:std_gradient], std_gradient[maxIter])\n", + "\n", + "end\n", + "\n", + "\n", + "CSV.write(outputvsc, accData)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.9.3", + "language": "julia", + "name": "julia-1.9" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.9.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/project/testing/L-BFGS/LBFGS_3D.ipynb b/project/testing/L-BFGS/LBFGS_3D.ipynb new file mode 100644 index 0000000..14473bb --- /dev/null +++ b/project/testing/L-BFGS/LBFGS_3D.ipynb @@ -0,0 +1,291 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "7b26750c-46f3-4650-aad9-759312cb7486", + "metadata": {}, + "outputs": [], + "source": [ + "include(\"../../L-BFGS/OracleFunction.jl\")\n", + "include(\"../../L-BFGS/LBFGS.jl\")\n", + "include(\"../../utilities/genFunc.jl\")\n", + "using .LBFGS\n", + "using .OracleFunction\n", + "using LinearAlgebra, BenchmarkTools, CSV, DataFrames\n", + "\n", + "baseDir = joinpath(\"../\", \"results/LBFGS/3D/\")\n", + "mkpath(baseDir);" + ] + }, + { + "cell_type": "markdown", + "id": "468b2db2", + "metadata": {}, + "source": [ + "## 3D PLOTS" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f96101fb-0029-48c9-b5f0-4b9b9f45feb6", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "### time with respect to lambda and epsilon\n", + "\n", + "# parameters\n", + "lambdas = 10. .^(-15:6)\n", + "epss = 10. .^(-15:-6)\n", + "memsize = 7\n", + "m = 300\n", + "n = 20\n", + "\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"statisticsLBFGS-lambda-eps-m\" * string(m) * \"n\" * string(n) * \"--time.csv\");\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :epsilon => Array{Float64}(undef, 0),\n", + " :memsize => Array{Int64}(undef, 0),\n", + " :meantime => Array{Float64}(undef, 0),\n", + " :stdtime => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "# computation\n", + "for (λ, ϵ) ∈ Iterators.product(lambdas, epss) |> collect\n", + " ls = genFunc(:exactRandDataset, λ=λ, m=m, n=n) |> LeastSquaresF\n", + "\n", + " t = @benchmark begin\n", + " _ = LimitedMemoryBFGS($ls, ϵ=$ϵ, m=memsize)\n", + " end samples=10 evals=1\n", + "\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:epsilon], ϵ)\n", + " push!(accData[:memsize], memsize)\n", + " push!(accData[:meantime], mean(t.times))\n", + " push!(accData[:stdtime], std(t.times))\n", + " println(\"Done: λ \" * string(λ) * \" - ϵ \" * string(ϵ))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5b36db4f-2659-4591-8640-cd5071788f67", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "### iterations with respect to lambda and epsilon\n", + "\n", + "# parameters\n", + "lambdas = 10. .^(-15:6)\n", + "epss = 10. .^(-15:-6)\n", + "memsize = 7\n", + "maxIter = 10000\n", + "m = 300\n", + "n = 20\n", + "\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"statisticsLBFGS-lambda-eps-m\" * string(m) * \"n\" * string(n) * \"--iterations.csv\");\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :epsilon => Array{Float64}(undef, 0),\n", + " :memsize => Array{Int64}(undef, 0),\n", + " :iterations => Array{Int64}(undef, 0)\n", + " )\n", + "\n", + "# computation\n", + "for (λ, ϵ) ∈ Iterators.product(lambdas, epss) |> collect\n", + " ls = genFunc(:exactRandDataset, λ=λ, m=m, n=n) |> LeastSquaresF\n", + "\n", + " t = maxIter - LimitedMemoryBFGS(ls, ϵ=ϵ, MaxEvaluations=maxIter, m=memsize)[:RemainingEvaluations]\n", + "\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:epsilon], ϵ)\n", + " push!(accData[:memsize], memsize)\n", + " push!(accData[:iterations], t)\n", + " println(\"Done: λ \" * string(λ) * \" - ϵ \" * string(ϵ))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3bb9382-8e68-49aa-81ac-6e3e3fcdf85b", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "### error with respect to lambda and epsilon\n", + "\n", + "# parameters\n", + "lambdas = 10. .^(-15:6)\n", + "epss = 10. .^(-15:-6)\n", + "memsize = 7\n", + "maxIter = 10000\n", + "m = 300\n", + "n = 20\n", + "\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"statisticsLBFGS-lambda-eps-m\" * string(m) * \"n\" * string(n) * \"--error.csv\");\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :epsilon => Array{Float64}(undef, 0),\n", + " :memsize => Array{Int64}(undef, 0),\n", + " :relative => Array{Float64}(undef, 0),\n", + " :residual => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "# computation\n", + "for (λ, ϵ) ∈ Iterators.product(lambdas, epss) |> collect\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " t = LimitedMemoryBFGS(ls, ϵ=ϵ, MaxEvaluations=maxIter, m=memsize)\n", + "\n", + " relative_error = norm(t[:x] - gf[:w_star]) / norm(gf[:w_star])\n", + " residual = norm(gf[:X_hat] * t[:x] - gf[:y_hat]) / norm(gf[:y_hat])\n", + "\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:epsilon], ϵ)\n", + " push!(accData[:memsize], memsize)\n", + " push!(accData[:relative], relative_error)\n", + " push!(accData[:residual], residual)\n", + " println(\"Done: λ \" * string(λ) * \" - ϵ \" * string(ϵ))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cff97997-96a4-4223-bed9-00663a9a2a6c", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "### iterations with respect to epsilon and memory size\n", + "\n", + "# parameters\n", + "λ = 10^-3\n", + "epss = 10. .^(-15:-6)\n", + "memsizes = [1, 3, 5, 7, 9, 11, 15, 20, 40, 60]\n", + "maxIter = 10000\n", + "m = 300\n", + "n = 20\n", + "\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"statisticsLBFGS-eps-mem-m\" * string(m) * \"n\" * string(n) * \"--iterations.csv\");\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :epsilon => Array{Float64}(undef, 0),\n", + " :memsize => Array{Int64}(undef, 0),\n", + " :iterations => Array{Int64}(undef, 0)\n", + " )\n", + "\n", + "# computation\n", + "for (ϵ, memsize) ∈ Iterators.product(epss, memsizes) |> collect\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " t = maxIter - LimitedMemoryBFGS(ls, ϵ=ϵ, MaxEvaluations=maxIter, m=memsize)[:RemainingEvaluations]\n", + "\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:epsilon], ϵ)\n", + " push!(accData[:memsize], memsize)\n", + " push!(accData[:iterations], t)\n", + " println(\"Done: mem \" * string(memsize) * \" - ϵ \" * string(ϵ))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "65a5b24b-ad25-4f13-8ebc-433e5e91eb75", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "### error with respect to epsilon and memory size\n", + "\n", + "# parameters\n", + "λ = 10^-3\n", + "epss = 10. .^(-15:-6)\n", + "memsizes = [1, 3, 5, 7, 9, 11, 15, 20, 40, 60]\n", + "maxIter = 10000\n", + "m = 300\n", + "n = 20\n", + "\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"statisticsLBFGS-eps-mem-m\" * string(m) * \"n\" * string(n) * \"--error.csv\");\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :epsilon => Array{Float64}(undef, 0),\n", + " :memsize => Array{Int64}(undef, 0),\n", + " :relative => Array{Float64}(undef, 0),\n", + " :residual => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "# computation\n", + "for (ϵ, memsize) ∈ Iterators.product(epss, memsizes) |> collect\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " t = LimitedMemoryBFGS(ls, ϵ=ϵ, MaxEvaluations=maxIter, m=memsize)\n", + "\n", + " relative_error = norm(t[:x] - gf[:w_star]) / norm(gf[:w_star])\n", + " residual = norm(gf[:X_hat] * t[:x] - gf[:y_hat]) / norm(gf[:y_hat])\n", + "\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:epsilon], ϵ)\n", + " push!(accData[:memsize], memsize)\n", + " push!(accData[:relative], relative_error)\n", + " push!(accData[:residual], residual)\n", + " println(\"Done: mem \" * string(memsize) * \" - ϵ \" * string(ϵ))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.9.3", + "language": "julia", + "name": "julia-1.9" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.9.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/project/testing/L-BFGS/LBFGS_ill.ipynb b/project/testing/L-BFGS/LBFGS_ill.ipynb new file mode 100644 index 0000000..1a5fc65 --- /dev/null +++ b/project/testing/L-BFGS/LBFGS_ill.ipynb @@ -0,0 +1,240 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "include(\"../../L-BFGS/OracleFunction.jl\")\n", + "include(\"../../L-BFGS/LBFGS.jl\")\n", + "include(\"../../utilities/genFunc.jl\")\n", + "using .LBFGS\n", + "using .OracleFunction\n", + "using LinearAlgebra, BenchmarkTools, CSV, DataFrames\n", + "\n", + "baseDir = joinpath(\"../\", \"results/LBFGS/ill_conditioned/\")\n", + "mkpath(baseDir);" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LeastSquaresF{Float64, Main.OracleFunction.var\"#f#1\"{Matrix{Float64}, Vector{Float64}}, Main.OracleFunction.var\"#df#2\"{Matrix{Float64}, Vector{Float64}}}(OracleF{Float64, Main.OracleFunction.var\"#f#1\"{Matrix{Float64}, Vector{Float64}}, Main.OracleFunction.var\"#df#2\"{Matrix{Float64}, Vector{Float64}}}([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 … 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], Main.OracleFunction.var\"#f#1\"{Matrix{Float64}, Vector{Float64}}([0.2592586022640595 0.6963535943217778 … 0.2686596970120587 -0.36678046257922436; 0.514702105566236 -0.7473274556601393 … 0.19887814981786733 0.8663661527822621; … ; 0.0 0.0 … 1.0e-12 0.0; 0.0 0.0 … 0.0 1.0e-12], [7.689547521112595, -23.188865118086323, 13.727391619018281, -4.242513852908478, 23.90743053982707, 1.8343747758964255, -1.159884201591351, 2.216694337124031, 2.730975021441503, -19.180979473592664 … -6.088266825619777e-13, 1.1943252953272254e-13, 4.048371886383304e-13, -6.926115820555791e-13, 3.8752674617747763e-13, 6.919446457660259e-13, 5.147174879596801e-13, -5.887351028568868e-13, 6.537120155767028e-13, 3.946146858948962e-13]), Main.OracleFunction.var\"#df#2\"{Matrix{Float64}, Vector{Float64}}([0.2592586022640595 0.6963535943217778 … 0.2686596970120587 -0.36678046257922436; 0.514702105566236 -0.7473274556601393 … 0.19887814981786733 0.8663661527822621; … ; 0.0 0.0 … 1.0e-12 0.0; 0.0 0.0 … 0.0 1.0e-12], [7.689547521112595, -23.188865118086323, 13.727391619018281, -4.242513852908478, 23.90743053982707, 1.8343747758964255, -1.159884201591351, 2.216694337124031, 2.730975021441503, -19.180979473592664 … -6.088266825619777e-13, 1.1943252953272254e-13, 4.048371886383304e-13, -6.926115820555791e-13, 3.8752674617747763e-13, 6.919446457660259e-13, 5.147174879596801e-13, -5.887351028568868e-13, 6.537120155767028e-13, 3.946146858948962e-13])), [0.2592586022640595 0.6963535943217778 … 0.2686596970120587 -0.36678046257922436; 0.514702105566236 -0.7473274556601393 … 0.19887814981786733 0.8663661527822621; … ; 0.0 0.0 … 1.0e-12 0.0; 0.0 0.0 … 0.0 1.0e-12], [7.689547521112595, -23.188865118086323, 13.727391619018281, -4.242513852908478, 23.90743053982707, 1.8343747758964255, -1.159884201591351, 2.216694337124031, 2.730975021441503, -19.180979473592664 … -6.088266825619777e-13, 1.1943252953272254e-13, 4.048371886383304e-13, -6.926115820555791e-13, 3.8752674617747763e-13, 6.919446457660259e-13, 5.147174879596801e-13, -5.887351028568868e-13, 6.537120155767028e-13, 3.946146858948962e-13], [6.711447540940819 -2.307272578298896 … 1.2541719054073133 -1.8639946218858157; -2.307272578298896 8.480411930046433 … -1.627313075365743 0.5078496705429104; … ; 1.2541719054073133 -1.627313075365743 … 7.4470442813402595 -4.4848817373554315; -1.8639946218858157 0.5078496705429104 … -4.4848817373554315 6.7106397848856], [-3.33043813332588 57.71972704197855 … 26.336102875181652 -62.14554952332052])" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# parameters for exact LS vs AWLS vs quadratic function \n", + "λ = 10^-12\n", + "ϵ = 10^-14\n", + "memsize = 7\n", + "maxIters = 1:200\n", + "m = 1000\n", + "n = 20\n", + "num_trials = 20\n", + "\n", + "gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + "non_quadratic = OracleF(ones(size(gf[:X_hat], 2)),\n", + " (x) -> norm(gf[:X_hat] * x - gf[:y_hat]),\n", + " (x) -> inv(norm(gf[:X_hat] * x - gf[:y_hat])) * gf[:X_hat]' * (gf[:X_hat] * x - gf[:y_hat])\n", + " )\n", + "quadratic = OracleF(ones(size(gf[:X_hat], 2)),\n", + " (x) -> norm(gf[:X_hat] * x - gf[:y_hat])^2,\n", + " (x) -> 2 * gf[:X_hat]' * (gf[:X_hat] * x - gf[:y_hat])\n", + " )\n", + "ls = LeastSquaresF(gf)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done trial 1\n", + "Done trial 2\n", + "Done trial 3\n", + "Done trial 4\n", + "Done trial 5\n", + "Done trial 6\n", + "Done trial 7\n", + "Done trial 8\n", + "Done trial 9\n", + "Done trial 10\n", + "Done trial 11\n", + "Done trial 12\n", + "Done trial 13\n", + "Done trial 14\n", + "Done trial 15\n", + "Done trial 16\n", + "Done trial 17\n", + "Done trial 18\n", + "Done trial 19\n", + "Done trial 20\n" + ] + }, + { + "data": { + "text/plain": [ + "\"../results/LBFGS/ill_conditioned/statisticsLBFGS-iterations-m1000n20--error-norm.csv\"" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "### residual, relative error and norm of gradient with respect to iterations with exact line search\n", + "using Statistics, CSV\n", + "\n", + "# Preallocate arrays\n", + "num_iterations = length(maxIters)\n", + "\n", + "gradients = zeros(num_trials, num_iterations)\n", + "residuals = zeros(num_trials, num_iterations)\n", + "relative_errors = zeros(num_trials, num_iterations)\n", + "\n", + "for trial in 1:num_trials\n", + "\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " for (i, maxIter) in enumerate(maxIters)\n", + " t = LimitedMemoryBFGS(ls, ϵ=ϵ, MaxEvaluations=maxIter, m=memsize)\n", + "\n", + " relative_errors[trial, i] = norm(t[:x] - gf[:w_star]) / norm(gf[:w_star])\n", + " residuals[trial, i] = norm(gf[:X_hat] * t[:x] - gf[:y_hat]) / norm(gf[:y_hat])\n", + " gradients[trial, i] = norm(t[:grad])\n", + " end\n", + "\n", + " println(\"Done trial \", trial)\n", + " \n", + "end\n", + "\n", + "# Calculate mean and standard deviation\n", + "mean_relative = mean(relative_errors, dims=1)'\n", + "std_relative = std(relative_errors, dims=1)'\n", + "mean_residual = mean(residuals, dims=1)'\n", + "std_residual = std(residuals, dims=1)'\n", + "mean_gradient = mean(gradients, dims=1)'\n", + "std_gradient = std(gradients, dims=1)'\n", + "\n", + "\n", + "# Write results to CSV\n", + "outputvsc = joinpath(baseDir, \"statisticsLBFGS-iterations-m\" * string(m) * \"n\" * string(n) * \"--error-norm.csv\");\n", + "\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :epsilon => Array{Float64}(undef, 0),\n", + " :memsize => Array{Int64}(undef, 0),\n", + " :maxiterations => Array{Int64}(undef, 0),\n", + " :mean_relative => Array{Float64}(undef, 0),\n", + " :std_relative => Array{Float64}(undef, 0),\n", + " :mean_residual => Array{Float64}(undef, 0),\n", + " :std_residual => Array{Float64}(undef, 0),\n", + " :mean_gradient => Array{Float64}(undef, 0),\n", + " :std_gradient => Array{Float64}(undef, 0)\n", + ")\n", + "\n", + "# create dataframe with columns from arrays\n", + "for maxIter ∈ maxIters\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:epsilon], ϵ)\n", + " push!(accData[:memsize], memsize)\n", + " push!(accData[:maxiterations], maxIter)\n", + " push!(accData[:mean_relative], mean_relative[maxIter])\n", + " push!(accData[:std_relative], std_relative[maxIter])\n", + " push!(accData[:mean_residual], mean_residual[maxIter])\n", + " push!(accData[:std_residual], std_residual[maxIter])\n", + " push!(accData[:mean_gradient], mean_gradient[maxIter])\n", + " push!(accData[:std_gradient], std_gradient[maxIter])\n", + "\n", + "end\n", + "\n", + "\n", + "CSV.write(outputvsc, accData)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "### relative error with respect to iterations and memeory size\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"statisticsLBFGS-iterations-m\" * string(m) * \"n\" * string(n) * \"--memsize.csv\");\n", + "accData = Dict(\n", + "\n", + "\n", + " :memsize => Array{Int64}(undef, 0),\n", + " :iterations => Array{Int64}(undef, 0),\n", + " :maxiterations => Array{Int64}(undef, 0),\n", + " :relative => Array{Float64}(undef, 0),\n", + " :residual => Array{Float64}(undef, 0),\n", + " :gradient => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "memsizes = [1 3 5 7 9 11]\n", + "\n", + "for memsize ∈ memsizes\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + " ls = LeastSquaresF(gf)\n", + " for maxIter ∈ maxIters\n", + "\n", + " t = LimitedMemoryBFGS(ls, ϵ=ϵ, MaxEvaluations=maxIter, m=memsize)\n", + "\n", + " relative_error = norm(t[:x] - gf[:w_star]) / norm(gf[:w_star])\n", + " residual = norm(gf[:X_hat] * t[:x] - gf[:y_hat]) / norm(gf[:y_hat])\n", + "\n", + " iterations = maxIter - t[:RemainingEvaluations]\n", + "\n", + " push!(accData[:memsize], memsize)\n", + " push!(accData[:iterations], iterations)\n", + " push!(accData[:maxiterations], maxIter)\n", + " push!(accData[:relative], relative_error)\n", + " push!(accData[:residual], residual)\n", + " push!(accData[:gradient], norm(t[:grad]))\n", + " flush(stdout)\n", + " end\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.9.3", + "language": "julia", + "name": "julia-1.9" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.9.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/project/testing/L-BFGS/LBFGS_well.ipynb b/project/testing/L-BFGS/LBFGS_well.ipynb new file mode 100644 index 0000000..91b4bda --- /dev/null +++ b/project/testing/L-BFGS/LBFGS_well.ipynb @@ -0,0 +1,1678 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "include(\"../../L-BFGS/OracleFunction.jl\")\n", + "include(\"../../L-BFGS/LBFGS.jl\")\n", + "include(\"../../utilities/genFunc.jl\")\n", + "using .LBFGS\n", + "using .OracleFunction\n", + "using LinearAlgebra, BenchmarkTools, CSV, DataFrames\n", + "\n", + "baseDir = joinpath(\"../\", \"results/LBFGS/well_conditioned/\")\n", + "mkpath(baseDir);" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LeastSquaresF{Float64, Main.OracleFunction.var\"#f#1\"{Matrix{Float64}, Vector{Float64}}, Main.OracleFunction.var\"#df#2\"{Matrix{Float64}, Vector{Float64}}}(OracleF{Float64, Main.OracleFunction.var\"#f#1\"{Matrix{Float64}, Vector{Float64}}, Main.OracleFunction.var\"#df#2\"{Matrix{Float64}, Vector{Float64}}}([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 … 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], Main.OracleFunction.var\"#f#1\"{Matrix{Float64}, Vector{Float64}}([-0.08250861069928206 0.17852719210948242 … 0.19042005909630877 -0.507902809042877; -0.5572519727292988 0.715127783454649 … -0.9011645602213216 0.038321040360024705; … ; 0.0 0.0 … 0.0001 0.0; 0.0 0.0 … 0.0 0.0001], [-7.9909737306123745, 14.69538119428254, 2.4593509793825494, -3.953587686159141, 4.252647143131211, -4.911100267400292, 18.00878115460575, -11.083562830390674, 7.4308455637609505, 1.3130519694521448 … 6.998564939162414e-5, 3.056340701582987e-5, -2.4008979560648935e-5, -9.410492147107272e-5, 2.467191606780468e-5, 4.96447234009942e-5, 4.8914799569796345e-5, -4.4946452188730815e-5, -3.245306940451023e-5, -8.167470156764867e-5]), Main.OracleFunction.var\"#df#2\"{Matrix{Float64}, Vector{Float64}}([-0.08250861069928206 0.17852719210948242 … 0.19042005909630877 -0.507902809042877; -0.5572519727292988 0.715127783454649 … -0.9011645602213216 0.038321040360024705; … ; 0.0 0.0 … 0.0001 0.0; 0.0 0.0 … 0.0 0.0001], [-7.9909737306123745, 14.69538119428254, 2.4593509793825494, -3.953587686159141, 4.252647143131211, -4.911100267400292, 18.00878115460575, -11.083562830390674, 7.4308455637609505, 1.3130519694521448 … 6.998564939162414e-5, 3.056340701582987e-5, -2.4008979560648935e-5, -9.410492147107272e-5, 2.467191606780468e-5, 4.96447234009942e-5, 4.8914799569796345e-5, -4.4946452188730815e-5, -3.245306940451023e-5, -8.167470156764867e-5])), [-0.08250861069928206 0.17852719210948242 … 0.19042005909630877 -0.507902809042877; -0.5572519727292988 0.715127783454649 … -0.9011645602213216 0.038321040360024705; … ; 0.0 0.0 … 0.0001 0.0; 0.0 0.0 … 0.0 0.0001], [-7.9909737306123745, 14.69538119428254, 2.4593509793825494, -3.953587686159141, 4.252647143131211, -4.911100267400292, 18.00878115460575, -11.083562830390674, 7.4308455637609505, 1.3130519694521448 … 6.998564939162414e-5, 3.056340701582987e-5, -2.4008979560648935e-5, -9.410492147107272e-5, 2.467191606780468e-5, 4.96447234009942e-5, 4.8914799569796345e-5, -4.4946452188730815e-5, -3.245306940451023e-5, -8.167470156764867e-5], [5.64238586295001 -1.4161105353595125 … -0.24000848265428637 2.2443247726411486; -1.4161105353595125 9.47068958668655 … -0.15679652316033219 -1.8269571350735225; … ; -0.24000848265428637 -0.15679652316033219 … 6.404915810836336 1.0752224296359685; 2.2443247726411486 -1.8269571350735225 … 1.0752224296359685 4.262488495871844], [-58.363374244372 33.966688207333114 … 11.642825309750123 -29.15403793180277])" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# parameters for exact LS vs AWLS vs quadratic function \n", + "λ = 10^-4\n", + "ϵ = 10^-14\n", + "memsize = 7\n", + "maxIters = 1:200\n", + "m = 1000\n", + "n = 20\n", + "num_trials = 20\n", + "\n", + "gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + "non_quadratic = OracleF(ones(size(gf[:X_hat], 2)),\n", + " (x) -> norm(gf[:X_hat] * x - gf[:y_hat]),\n", + " (x) -> inv(norm(gf[:X_hat] * x - gf[:y_hat])) * gf[:X_hat]' * (gf[:X_hat] * x - gf[:y_hat])\n", + " )\n", + "quadratic = OracleF(ones(size(gf[:X_hat], 2)),\n", + " (x) -> norm(gf[:X_hat] * x - gf[:y_hat]) ^ 2,\n", + " (x) -> 2 * gf[:X_hat]' * (gf[:X_hat] * x - gf[:y_hat])\n", + " )\n", + "ls = LeastSquaresF(gf)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Exact Line Search vs AWLS vs AWLS non quadratic" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done trial 1\n", + "Done trial 2\n", + "Done trial 3\n", + "Done trial 4\n", + "Done trial 5\n", + "Done trial 6\n", + "Done trial 7\n", + "Done trial 8\n", + "Done trial 9\n", + "Done trial 10\n", + "Done trial 11\n", + "Done trial 12\n", + "Done trial 13\n", + "Done trial 14\n", + "Done trial 15\n", + "Done trial 16\n", + "Done trial 17\n", + "Done trial 18\n", + "Done trial 19\n", + "Done trial 20\n" + ] + }, + { + "data": { + "text/plain": [ + "\"../results/LBFGS/well_conditioned/statisticsLBFGS-iterations-m1000n20--error-norm.csv\"" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "### residual, relative error and norm of gradient with respect to iterations with exact line search\n", + "using Statistics, CSV\n", + "\n", + "# Preallocate arrays\n", + "num_iterations = length(maxIters)\n", + "\n", + "gradients = zeros(num_trials, num_iterations)\n", + "residuals = zeros(num_trials, num_iterations)\n", + "relative_errors = zeros(num_trials, num_iterations)\n", + "\n", + "for trial in 1:num_trials\n", + "\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " for (i, maxIter) in enumerate(maxIters)\n", + " t = LimitedMemoryBFGS(ls, ϵ=ϵ, MaxEvaluations=maxIter, m=memsize)\n", + "\n", + " relative_errors[trial, i] = norm(t[:x] - gf[:w_star]) / norm(gf[:w_star])\n", + " residuals[trial, i] = norm(gf[:X_hat] * t[:x] - gf[:y_hat]) / norm(gf[:y_hat])\n", + " gradients[trial, i] = norm(t[:grad])\n", + " end\n", + "\n", + " println(\"Done trial \", trial)\n", + " \n", + "end\n", + "\n", + "# Calculate mean and standard deviation\n", + "mean_relative = mean(relative_errors, dims=1)'\n", + "std_relative = std(relative_errors, dims=1)'\n", + "mean_residual = mean(residuals, dims=1)'\n", + "std_residual = std(residuals, dims=1)'\n", + "mean_gradient = mean(gradients, dims=1)'\n", + "std_gradient = std(gradients, dims=1)'\n", + "\n", + "\n", + "# Write results to CSV\n", + "outputvsc = joinpath(baseDir, \"statisticsLBFGS-iterations-m\" * string(m) * \"n\" * string(n) * \"--error-norm.csv\");\n", + "\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :epsilon => Array{Float64}(undef, 0),\n", + " :memsize => Array{Int64}(undef, 0),\n", + " :maxiterations => Array{Int64}(undef, 0),\n", + " :mean_relative => Array{Float64}(undef, 0),\n", + " :std_relative => Array{Float64}(undef, 0),\n", + " :mean_residual => Array{Float64}(undef, 0),\n", + " :std_residual => Array{Float64}(undef, 0),\n", + " :mean_gradient => Array{Float64}(undef, 0),\n", + " :std_gradient => Array{Float64}(undef, 0)\n", + ")\n", + "\n", + "# create dataframe with columns from arrays\n", + "for maxIter ∈ maxIters\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:epsilon], ϵ)\n", + " push!(accData[:memsize], memsize)\n", + " push!(accData[:maxiterations], maxIter)\n", + " push!(accData[:mean_relative], mean_relative[maxIter])\n", + " push!(accData[:std_relative], std_relative[maxIter])\n", + " push!(accData[:mean_residual], mean_residual[maxIter])\n", + " push!(accData[:std_residual], std_residual[maxIter])\n", + " push!(accData[:mean_gradient], mean_gradient[maxIter])\n", + " push!(accData[:std_gradient], std_gradient[maxIter])\n", + "\n", + "end\n", + "\n", + "\n", + "CSV.write(outputvsc, accData)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done trial 1\n", + "Done trial 2\n", + "Done trial 3\n", + "Done trial 4\n", + "Done trial 5\n", + "Done trial 6\n", + "Done trial 7\n", + "Done trial 8\n", + "Done trial 9\n", + "Done trial 10\n", + "Done trial 11\n", + "Done trial 12\n", + "Done trial 13\n", + "Done trial 14\n", + "Done trial 15\n", + "Done trial 16\n", + "Done trial 17\n", + "Done trial 18\n", + "Done trial 19\n", + "Done trial 20\n" + ] + }, + { + "data": { + "text/plain": [ + "\"../results/LBFGS/well_conditioned/statisticsLBFGS-AWLS-iterations-m1000n20--error-norm.csv\"" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "using Statistics, CSV\n", + "\n", + "# Preallocate arrays\n", + "num_iterations = length(maxIters)\n", + "\n", + "\n", + "gradients = zeros(num_trials, num_iterations)\n", + "residuals = zeros(num_trials, num_iterations)\n", + "relative_errors = zeros(num_trials, num_iterations)\n", + "\n", + "\n", + "for trial in 1:num_trials\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + " non_quadratic = OracleF(ones(size(gf[:X_hat], 2)),\n", + " (x) -> norm(gf[:X_hat] * x - gf[:y_hat]),\n", + " (x) -> inv(norm(gf[:X_hat] * x - gf[:y_hat])) * gf[:X_hat]' * (gf[:X_hat] * x - gf[:y_hat]))\n", + "\n", + " for (i, maxIter) in enumerate(maxIters)\n", + " t = LimitedMemoryBFGS(non_quadratic, ϵ=ϵ, MaxEvaluations=maxIter, m=memsize)\n", + "\n", + " relative_errors[trial, i] = norm(t[:x] - gf[:w_star]) / norm(gf[:w_star])\n", + " residuals[trial, i] = norm(gf[:X_hat] * t[:x] - gf[:y_hat]) / norm(gf[:y_hat])\n", + " gradients[trial, i] = norm(t[:grad])\n", + " end\n", + "\n", + " println(\"Done trial \", trial)\n", + " \n", + "end\n", + "\n", + "mean_relative = mean(relative_errors, dims=1)'\n", + "std_relative = std(relative_errors, dims=1)'\n", + "mean_residual = mean(residuals, dims=1)'\n", + "std_residual = std(residuals, dims=1)'\n", + "mean_gradient = mean(gradients, dims=1)'\n", + "std_gradient = std(gradients, dims=1)'\n", + "\n", + "# Write results to CSV\n", + "outputvsc = joinpath(baseDir, \"statisticsLBFGS-AWLS-iterations-m\" * string(m) * \"n\" * string(n) * \"--error-norm.csv\")\n", + "\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :epsilon => Array{Float64}(undef, 0),\n", + " :memsize => Array{Int64}(undef, 0),\n", + " :maxiterations => Array{Int64}(undef, 0),\n", + " :mean_relative => Array{Float64}(undef, 0),\n", + " :std_relative => Array{Float64}(undef, 0),\n", + " :mean_residual => Array{Float64}(undef, 0),\n", + " :std_residual => Array{Float64}(undef, 0),\n", + " :mean_gradient => Array{Float64}(undef, 0),\n", + " :std_gradient => Array{Float64}(undef, 0)\n", + ")\n", + "\n", + "\n", + "for maxIter ∈ maxIters\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:epsilon], ϵ)\n", + " push!(accData[:memsize], memsize)\n", + " push!(accData[:maxiterations], maxIter)\n", + " push!(accData[:mean_relative], mean_relative[maxIter])\n", + " push!(accData[:std_relative], std_relative[maxIter])\n", + " push!(accData[:mean_residual], mean_residual[maxIter])\n", + " push!(accData[:std_residual], std_residual[maxIter])\n", + " push!(accData[:mean_gradient], mean_gradient[maxIter])\n", + " push!(accData[:std_gradient], std_gradient[maxIter])\n", + "\n", + "end\n", + "\n", + "\n", + "CSV.write(outputvsc, accData)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done trial 1\n", + "Done trial 2\n", + "Done trial 3\n", + "Done trial 4\n", + "Done trial 5\n", + "Done trial 6\n", + "Done trial 7\n", + "Done trial 8\n", + "Done trial 9\n", + "Done trial 10\n", + "Done trial 11\n", + "Done trial 12\n", + "Done trial 13\n", + "Done trial 14\n", + "Done trial 15\n", + "Done trial 16\n", + "Done trial 17\n", + "Done trial 18\n", + "Done trial 19\n", + "Done trial 20\n" + ] + }, + { + "data": { + "text/plain": [ + "\"../results/LBFGS/well_conditioned/statisticsLBFGS-AWLS-quad-iterations-m1000n20--error-norm.csv\"" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "### error and norm of gradient with respect to iterations with AWLS and quadratic function\n", + "# output csv\n", + "using Statistics, CSV\n", + "\n", + "maxIters = 1:400\n", + "num_trials = 5\n", + "# Preallocate arrays\n", + "num_iterations = length(maxIters)\n", + "\n", + "gradients = zeros(num_trials, num_iterations)\n", + "residuals = zeros(num_trials, num_iterations)\n", + "relative_errors = zeros(num_trials, num_iterations)\n", + "iterations = zeros(num_trials, num_iterations)\n", + "\n", + "for trial in 1:num_trials\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + " quadratic = OracleF(ones(size(gf[:X_hat], 2)),\n", + " (x) -> norm(gf[:X_hat] * x - gf[:y_hat]) ^ 2,\n", + " (x) -> 2 * gf[:X_hat]' * (gf[:X_hat] * x - gf[:y_hat]))\n", + "\n", + " for (i, maxIter) in enumerate(maxIters)\n", + " t = LimitedMemoryBFGS(quadratic, ϵ=ϵ, MaxEvaluations=maxIter, m=memsize)\n", + "\n", + "\n", + " relative_errors[trial, i] = norm(t[:x] - gf[:w_star]) / norm(gf[:w_star])\n", + " residuals[trial, i] = norm(gf[:X_hat] * t[:x] - gf[:y_hat]) / norm(gf[:y_hat])\n", + " gradients[trial, i] = norm(t[:grad])\n", + " iterations[trial, i] = i \n", + " end\n", + "\n", + " println(\"Done trial \", trial)\n", + " \n", + "end\n", + "\n", + "\n", + "mean_relative = mean(relative_errors, dims=1)'\n", + "std_relative = std(relative_errors, dims=1)'\n", + "mean_residual = mean(residuals, dims=1)'\n", + "std_residual = std(residuals, dims=1)'\n", + "mean_gradient = mean(gradients, dims=1)'\n", + "std_gradient = std(gradients, dims=1)'\n", + "\n", + "# Write results to CSV\n", + "outputvsc = joinpath(baseDir, \"statisticsLBFGS-AWLS-quad-iterations-m\" * string(m) * \"n\" * string(n) * \"--error-norm.csv\");\n", + "\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :epsilon => Array{Float64}(undef, 0),\n", + " :memsize => Array{Int64}(undef, 0),\n", + " :maxiterations => Array{Int64}(undef, 0),\n", + " :mean_relative => Array{Float64}(undef, 0),\n", + " :std_relative => Array{Float64}(undef, 0),\n", + " :mean_residual => Array{Float64}(undef, 0),\n", + " :std_residual => Array{Float64}(undef, 0),\n", + " :mean_gradient => Array{Float64}(undef, 0),\n", + " :std_gradient => Array{Float64}(undef, 0)\n", + ")\n", + "\n", + "\n", + "for maxIter ∈ maxIters\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:epsilon], ϵ)\n", + " push!(accData[:memsize], memsize)\n", + " push!(accData[:maxiterations], maxIter)\n", + " push!(accData[:mean_relative], mean_relative[maxIter])\n", + " push!(accData[:std_relative], std_relative[maxIter])\n", + " push!(accData[:mean_residual], mean_residual[maxIter])\n", + " push!(accData[:std_residual], std_residual[maxIter])\n", + " push!(accData[:mean_gradient], mean_gradient[maxIter])\n", + " push!(accData[:std_gradient], std_gradient[maxIter])\n", + "\n", + "end\n", + "\n", + "maxIters = 1:200\n", + "CSV.write(outputvsc, accData)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done: maxIter 1 -> error 1.9723953204307332\n", + "Done: maxIter 3 -> error 1.939327681829027\n", + "Done: maxIter 3 -> error 1.939327681829027\n", + "Done: maxIter 5 -> error 1.938744601524112\n", + "Done: maxIter 5 -> error 1.938744601524112\n", + "Done: maxIter 7 -> error 1.9387301726555504\n", + "Done: maxIter 7 -> error 1.9387301726555504\n", + "Done: maxIter 9 -> error 1.938729924452633\n", + "Done: maxIter 9 -> error 1.938729924452633\n", + "Done: maxIter 11 -> error 1.9387299211343605\n", + "Done: maxIter 11 -> error 1.9387299211343605\n", + "Done: maxIter 13 -> error 1.9387299209849849\n", + "Done: maxIter 13 -> error 1.9387299209849849\n", + "Done: maxIter 15 -> error 1.9387299209279898\n", + "Done: maxIter 15 -> error 1.9387299209279898\n", + "Done: maxIter 17 -> error 1.9387299208702156\n", + "Done: maxIter 17 -> error 1.9387299208702156\n", + "Done: maxIter 19 -> error 1.9387299208084985\n", + "Done: maxIter 19 -> error 1.9387299208084985\n", + "Done: maxIter 21 -> error 1.938729920747817\n", + "Done: maxIter 21 -> error 1.938729920747817\n", + "Done: maxIter 23 -> error 1.9387299206796087\n", + "Done: maxIter 23 -> error 1.9387299206796087\n", + "Done: maxIter 25 -> error 1.9387299199231465\n", + "Done: maxIter 25 -> error 1.9387299199231465\n", + "Done: maxIter 27 -> error 1.9387298127465629\n", + "Done: maxIter 27 -> error 1.9387298127465629\n", + "Done: maxIter 29 -> error 1.938714460442371\n", + "Done: maxIter 29 -> error 1.938714460442371\n", + "Done: maxIter 31 -> error 1.9374315002884304\n", + "Done: maxIter 31 -> error 1.9374315002884304\n", + "Done: maxIter 33 -> error 1.7562760231871746\n", + "Done: maxIter 33 -> error 1.7562760231871746\n", + "Done: maxIter 35 -> error 0.04356776086559559\n", + "Done: maxIter 35 -> error 0.04356776086559559\n", + "Done: maxIter 37 -> error 9.061973448650679e-5\n", + "Done: maxIter 37 -> error 9.061973448650679e-5\n", + "Done: maxIter 39 -> error 3.440621857510314e-7\n", + "Done: maxIter 39 -> error 3.440621857510314e-7\n", + "Done: maxIter 41 -> error 2.66014643973211e-7\n", + "Done: maxIter 41 -> error 2.66014643973211e-7\n", + "Done: maxIter 43 -> error 2.659769260338445e-7\n", + "Done: maxIter 43 -> error 2.659769260338445e-7\n", + "Done: maxIter 45 -> error 2.6597665194575154e-7\n", + "Done: maxIter 45 -> error 2.6597665194575154e-7\n", + "Done: maxIter 47 -> error 2.659766519335203e-7\n", + "Done: maxIter 47 -> error 2.659766519335203e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 49 -> error 2.659766519274138e-7\n", + "Done: maxIter 1 -> error 2.0049836277230204\n", + "Done: maxIter 3 -> error 1.988632699781604\n", + "Done: maxIter 3 -> error 1.988632699781604\n", + "Done: maxIter 5 -> error 1.988467129603326\n", + "Done: maxIter 5 -> error 1.988467129603326\n", + "Done: maxIter 7 -> error 1.9884657924887639\n", + "Done: maxIter 7 -> error 1.9884657924887639\n", + "Done: maxIter 9 -> error 1.988465773362678\n", + "Done: maxIter 9 -> error 1.988465773362678\n", + "Done: maxIter 11 -> error 1.9884657730375739\n", + "Done: maxIter 11 -> error 1.9884657730375739\n", + "Done: maxIter 13 -> error 1.9884657729668933\n", + "Done: maxIter 13 -> error 1.9884657729668933\n", + "Done: maxIter 15 -> error 1.9884657729096369\n", + "Done: maxIter 15 -> error 1.9884657729096369\n", + "Done: maxIter 17 -> error 1.9884657728456172\n", + "Done: maxIter 17 -> error 1.9884657728456172\n", + "Done: maxIter 19 -> error 1.9884657727871813\n", + "Done: maxIter 19 -> error 1.9884657727871813\n", + "Done: maxIter 21 -> error 1.9884657727225636\n", + "Done: maxIter 21 -> error 1.9884657727225636\n", + "Done: maxIter 23 -> error 1.9884657725441761\n", + "Done: maxIter 23 -> error 1.9884657725441761\n", + "Done: maxIter 25 -> error 1.9884657573424034\n", + "Done: maxIter 25 -> error 1.9884657573424034\n", + "Done: maxIter 27 -> error 1.9884645393770746\n", + "Done: maxIter 27 -> error 1.9884645393770746\n", + "Done: maxIter 29 -> error 1.9883372588576944\n", + "Done: maxIter 29 -> error 1.9883372588576944\n", + "Done: maxIter 31 -> error 1.9664499805231626\n", + "Done: maxIter 31 -> error 1.9664499805231626\n", + "Done: maxIter 33 -> error 0.2873831618377674\n", + "Done: maxIter 33 -> error 0.2873831618377674\n", + "Done: maxIter 35 -> error 0.0005907923555453286\n", + "Done: maxIter 35 -> error 0.0005907923555453286\n", + "Done: maxIter 37 -> error 1.6342698879255964e-6\n", + "Done: maxIter 37 -> error 1.6342698879255964e-6\n", + "Done: maxIter 39 -> error 6.195617992103437e-7\n", + "Done: maxIter 39 -> error 6.195617992103437e-7\n", + "Done: maxIter 41 -> error 6.195476990487086e-7\n", + "Done: maxIter 41 -> error 6.195476990487086e-7\n", + "Done: maxIter 43 -> error 6.195476990728299e-7\n", + "Done: maxIter 43 -> error 6.195476990728299e-7\n", + "Done: maxIter 45 -> error 6.195476990561281e-7\n", + "Done: maxIter 45 -> error 6.195476990561281e-7\n", + "Done: maxIter 47 -> error 6.195476990366841e-7\n", + "Done: maxIter 47 -> error 6.195476990366841e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 49 -> error 6.195476990222694e-7\n", + "Done: maxIter 1 -> error 1.9625598545277498\n", + "Done: maxIter 3 -> error 1.940073993890315\n", + "Done: maxIter 3 -> error 1.940073993890315\n", + "Done: maxIter 5 -> error 1.9396950016990628\n", + "Done: maxIter 5 -> error 1.9396950016990628\n", + "Done: maxIter 7 -> error 1.9396885223002964\n", + "Done: maxIter 7 -> error 1.9396885223002964\n", + "Done: maxIter 9 -> error 1.9396884469640865\n", + "Done: maxIter 9 -> error 1.9396884469640865\n", + "Done: maxIter 11 -> error 1.9396884457609136\n", + "Done: maxIter 11 -> error 1.9396884457609136\n", + "Done: maxIter 13 -> error 1.9396884456997905\n", + "Done: maxIter 13 -> error 1.9396884456997905\n", + "Done: maxIter 15 -> error 1.9396884456454386\n", + "Done: maxIter 15 -> error 1.9396884456454386\n", + "Done: maxIter 17 -> error 1.9396884455880379\n", + "Done: maxIter 17 -> error 1.9396884455880379\n", + "Done: maxIter 19 -> error 1.9396884455268275\n", + "Done: maxIter 19 -> error 1.9396884455268275\n", + "Done: maxIter 21 -> error 1.9396884454651087\n", + "Done: maxIter 21 -> error 1.9396884454651087\n", + "Done: maxIter 23 -> error 1.9396884453526213\n", + "Done: maxIter 23 -> error 1.9396884453526213\n", + "Done: maxIter 25 -> error 1.9396884340628109\n", + "Done: maxIter 25 -> error 1.9396884340628109\n", + "Done: maxIter 27 -> error 1.9396875131072022\n", + "Done: maxIter 27 -> error 1.9396875131072022\n", + "Done: maxIter 29 -> error 1.9391016061559911\n", + "Done: maxIter 29 -> error 1.9391016061559911\n", + "Done: maxIter 31 -> error 1.8621979883589765\n", + "Done: maxIter 31 -> error 1.8621979883589765\n", + "Done: maxIter 33 -> error 0.2390813654324172\n", + "Done: maxIter 33 -> error 0.2390813654324172\n", + "Done: maxIter 35 -> error 0.0007000385777335523\n", + "Done: maxIter 35 -> error 0.0007000385777335523\n", + "Done: maxIter 37 -> error 2.4102062589698386e-6\n", + "Done: maxIter 37 -> error 2.4102062589698386e-6\n", + "Done: maxIter 39 -> error 6.867118804226255e-7\n", + "Done: maxIter 39 -> error 6.867118804226255e-7\n", + "Done: maxIter 41 -> error 6.867041000457327e-7\n", + "Done: maxIter 41 -> error 6.867041000457327e-7\n", + "Done: maxIter 43 -> error 6.867041006129842e-7\n", + "Done: maxIter 43 -> error 6.867041006129842e-7\n", + "Done: maxIter 45 -> error 6.867041005919834e-7\n", + "Done: maxIter 45 -> error 6.867041005919834e-7\n", + "Done: maxIter 47 -> error 6.867041005721682e-7\n", + "Done: maxIter 47 -> error 6.867041005721682e-7\n", + "Done: maxIter 49 -> error 6.867041005511402e-7\n", + "Done: maxIter 49 -> error 6.867041005511402e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 51 -> error 6.867041005296422e-7\n", + "Done: maxIter 1 -> error 1.9525834026639726\n", + "Done: maxIter 3 -> error 1.931701155796341\n", + "Done: maxIter 3 -> error 1.931701155796341\n", + "Done: maxIter 5 -> error 1.931338839137954\n", + "Done: maxIter 5 -> error 1.931338839137954\n", + "Done: maxIter 7 -> error 1.9313339924151935\n", + "Done: maxIter 7 -> error 1.9313339924151935\n", + "Done: maxIter 9 -> error 1.931333912032809\n", + "Done: maxIter 9 -> error 1.931333912032809\n", + "Done: maxIter 11 -> error 1.9313339107963214\n", + "Done: maxIter 11 -> error 1.9313339107963214\n", + "Done: maxIter 13 -> error 1.931333910718522\n", + "Done: maxIter 13 -> error 1.931333910718522\n", + "Done: maxIter 15 -> error 1.931333910659593\n", + "Done: maxIter 15 -> error 1.931333910659593\n", + "Done: maxIter 17 -> error 1.9313339106026246\n", + "Done: maxIter 17 -> error 1.9313339106026246\n", + "Done: maxIter 19 -> error 1.9313339105420015\n", + "Done: maxIter 19 -> error 1.9313339105420015\n", + "Done: maxIter 21 -> error 1.9313339104807266\n", + "Done: maxIter 21 -> error 1.9313339104807266\n", + "Done: maxIter 23 -> error 1.9313339100439564\n", + "Done: maxIter 23 -> error 1.9313339100439564\n", + "Done: maxIter 25 -> error 1.9313338638200772\n", + "Done: maxIter 25 -> error 1.9313338638200772\n", + "Done: maxIter 27 -> error 1.9313300637909436\n", + "Done: maxIter 27 -> error 1.9313300637909436\n", + "Done: maxIter 29 -> error 1.9309323599289654\n", + "Done: maxIter 29 -> error 1.9309323599289654\n", + "Done: maxIter 31 -> error 1.8775187159054592\n", + "Done: maxIter 31 -> error 1.8775187159054592\n", + "Done: maxIter 33 -> error 0.08588898004301726\n", + "Done: maxIter 33 -> error 0.08588898004301726\n", + "Done: maxIter 35 -> error 5.8418062553067376e-5\n", + "Done: maxIter 35 -> error 5.8418062553067376e-5\n", + "Done: maxIter 37 -> error 3.333171381896593e-7\n", + "Done: maxIter 37 -> error 3.333171381896593e-7\n", + "Done: maxIter 39 -> error 3.2145990194905863e-7\n", + "Done: maxIter 39 -> error 3.2145990194905863e-7\n", + "Done: maxIter 41 -> error 3.2145989108633923e-7\n", + "Done: maxIter 41 -> error 3.2145989108633923e-7\n", + "Done: maxIter 43 -> error 3.214598910747803e-7\n", + "Done: maxIter 43 -> error 3.214598910747803e-7\n", + "Done: maxIter 45 -> error 3.2145989106052505e-7\n", + "Done: maxIter 45 -> error 3.2145989106052505e-7\n", + "Done: maxIter 47 -> error 3.214598910507003e-7\n", + "Done: maxIter 47 -> error 3.214598910507003e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 49 -> error 3.2145989103979646e-7\n", + "Done: maxIter 1 -> error 2.077251271914291\n", + "Done: maxIter 3 -> error 2.0398938246879914\n", + "Done: maxIter 3 -> error 2.0398938246879914\n", + "Done: maxIter 5 -> error 2.0393813910609246\n", + "Done: maxIter 5 -> error 2.0393813910609246\n", + "Done: maxIter 7 -> error 2.0393755776416573\n", + "Done: maxIter 7 -> error 2.0393755776416573\n", + "Done: maxIter 9 -> error 2.039375523039052\n", + "Done: maxIter 9 -> error 2.039375523039052\n", + "Done: maxIter 11 -> error 2.039375522427902\n", + "Done: maxIter 11 -> error 2.039375522427902\n", + "Done: maxIter 13 -> error 2.0393755223493577\n", + "Done: maxIter 13 -> error 2.0393755223493577\n", + "Done: maxIter 15 -> error 2.0393755222869263\n", + "Done: maxIter 15 -> error 2.0393755222869263\n", + "Done: maxIter 17 -> error 2.039375522222392\n", + "Done: maxIter 17 -> error 2.039375522222392\n", + "Done: maxIter 19 -> error 2.03937552215342\n", + "Done: maxIter 19 -> error 2.03937552215342\n", + "Done: maxIter 21 -> error 2.0393755220913574\n", + "Done: maxIter 21 -> error 2.0393755220913574\n", + "Done: maxIter 23 -> error 2.0393755220041996\n", + "Done: maxIter 23 -> error 2.0393755220041996\n", + "Done: maxIter 25 -> error 2.0393755178067203\n", + "Done: maxIter 25 -> error 2.0393755178067203\n", + "Done: maxIter 27 -> error 2.0393750405018864\n", + "Done: maxIter 27 -> error 2.0393750405018864\n", + "Done: maxIter 29 -> error 2.0392877714354234\n", + "Done: maxIter 29 -> error 2.0392877714354234\n", + "Done: maxIter 31 -> error 2.0301508869469127\n", + "Done: maxIter 31 -> error 2.0301508869469127\n", + "Done: maxIter 33 -> error 0.40812037071561236\n", + "Done: maxIter 33 -> error 0.40812037071561236\n", + "Done: maxIter 35 -> error 0.005297381514733345\n", + "Done: maxIter 35 -> error 0.005297381514733345\n", + "Done: maxIter 37 -> error 3.7898260149246722e-6\n", + "Done: maxIter 37 -> error 3.7898260149246722e-6\n", + "Done: maxIter 39 -> error 6.517588472793674e-7\n", + "Done: maxIter 39 -> error 6.517588472793674e-7\n", + "Done: maxIter 41 -> error 6.516702243186542e-7\n", + "Done: maxIter 41 -> error 6.516702243186542e-7\n", + "Done: maxIter 43 -> error 6.51670051976418e-7\n", + "Done: maxIter 43 -> error 6.51670051976418e-7\n", + "Done: maxIter 45 -> error 6.516700519564892e-7\n", + "Done: maxIter 45 -> error 6.516700519564892e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 47 -> error 6.516700519399299e-7\n", + "Done: maxIter 1 -> error 1.9934777767222684\n", + "Done: maxIter 3 -> error 1.9792819687148127\n", + "Done: maxIter 3 -> error 1.9792819687148127\n", + "Done: maxIter 5 -> error 1.9790268040343695\n", + "Done: maxIter 5 -> error 1.9790268040343695\n", + "Done: maxIter 7 -> error 1.9790238136186098\n", + "Done: maxIter 7 -> error 1.9790238136186098\n", + "Done: maxIter 9 -> error 1.979023764070804\n", + "Done: maxIter 9 -> error 1.979023764070804\n", + "Done: maxIter 11 -> error 1.9790237633918573\n", + "Done: maxIter 11 -> error 1.9790237633918573\n", + "Done: maxIter 13 -> error 1.9790237633226857\n", + "Done: maxIter 13 -> error 1.9790237633226857\n", + "Done: maxIter 15 -> error 1.9790237632639502\n", + "Done: maxIter 15 -> error 1.9790237632639502\n", + "Done: maxIter 17 -> error 1.979023763202223\n", + "Done: maxIter 17 -> error 1.979023763202223\n", + "Done: maxIter 19 -> error 1.9790237631435883\n", + "Done: maxIter 19 -> error 1.9790237631435883\n", + "Done: maxIter 21 -> error 1.9790237630818268\n", + "Done: maxIter 21 -> error 1.9790237630818268\n", + "Done: maxIter 23 -> error 1.9790237629922807\n", + "Done: maxIter 23 -> error 1.9790237629922807\n", + "Done: maxIter 25 -> error 1.9790237579609846\n", + "Done: maxIter 25 -> error 1.9790237579609846\n", + "Done: maxIter 27 -> error 1.9790232768901073\n", + "Done: maxIter 27 -> error 1.9790232768901073\n", + "Done: maxIter 29 -> error 1.978969128458866\n", + "Done: maxIter 29 -> error 1.978969128458866\n", + "Done: maxIter 31 -> error 1.9550941551906325\n", + "Done: maxIter 31 -> error 1.9550941551906325\n", + "Done: maxIter 33 -> error 0.31845722581197566\n", + "Done: maxIter 33 -> error 0.31845722581197566\n", + "Done: maxIter 35 -> error 0.0010319894682389618\n", + "Done: maxIter 35 -> error 0.0010319894682389618\n", + "Done: maxIter 37 -> error 2.4783559879855605e-6\n", + "Done: maxIter 37 -> error 2.4783559879855605e-6\n", + "Done: maxIter 39 -> error 5.756453696392672e-7\n", + "Done: maxIter 39 -> error 5.756453696392672e-7\n", + "Done: maxIter 41 -> error 5.75643559263135e-7\n", + "Done: maxIter 41 -> error 5.75643559263135e-7\n", + "Done: maxIter 43 -> error 5.756435592288391e-7\n", + "Done: maxIter 43 -> error 5.756435592288391e-7\n", + "Done: maxIter 45 -> error 5.756435592141896e-7\n", + "Done: maxIter 45 -> error 5.756435592141896e-7\n", + "Done: maxIter 47 -> error 5.756435591986304e-7\n", + "Done: maxIter 47 -> error 5.756435591986304e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n", + "Done: maxIter 49 -> error 5.756435591821421e-7\n" + ] + } + ], + "source": [ + "### relative error with repsepct to iterations and memory size\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"statisticsLBFGS-iterations-m\" * string(m) * \"n\" * string(n) * \"--memsize.csv\");\n", + "accData = Dict(\n", + " :memsize => Array{Int64}(undef, 0),\n", + " :iterations => Array{Int64}(undef, 0),\n", + " :maxiterations => Array{Int64}(undef, 0),\n", + " :relative => Array{Float64}(undef, 0),\n", + " :residual => Array{Float64}(undef, 0),\n", + " :gradient => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "memsizes = [1 3 5 7 9 11]\n", + "\n", + "for memsize ∈ memsizes\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + " ls = LeastSquaresF(gf)\n", + " for maxIter ∈ maxIters\n", + "\n", + " t = LimitedMemoryBFGS(ls, ϵ=ϵ, MaxEvaluations=maxIter, m=memsize)\n", + "\n", + " relative_error = norm(t[:x] - gf[:w_star]) / norm(gf[:w_star])\n", + " residual = norm(gf[:X_hat] * t[:x] - gf[:y_hat]) / norm(gf[:y_hat])\n", + "\n", + " iterations = maxIter - t[:RemainingEvaluations]\n", + "\n", + " push!(accData[:memsize], memsize)\n", + " push!(accData[:iterations], iterations)\n", + " push!(accData[:maxiterations], maxIter)\n", + " push!(accData[:relative], relative_error)\n", + " push!(accData[:residual], residual)\n", + " push!(accData[:gradient], norm(t[:grad]))\n", + " println(\"Done: maxIter \" * string(iterations) * \" -> error \" * string(relative_error))\n", + " flush(stdout)\n", + " end\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.9.3", + "language": "julia", + "name": "julia-1.9" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.9.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/project/testing/QR/QR.ipynb b/project/testing/QR/QR.ipynb new file mode 100644 index 0000000..ab7b2e8 --- /dev/null +++ b/project/testing/QR/QR.ipynb @@ -0,0 +1,176 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "57a2b335-a7f3-4ace-a2d8-52219c4febc5", + "metadata": {}, + "outputs": [], + "source": [ + "include(\"../../QR/housQR.jl\")\n", + "include(\"../../utilities/genFunc.jl\")\n", + "using .housQR\n", + "using LinearAlgebra, BenchmarkTools, CSV, DataFrames\n", + "\n", + "baseDir = joinpath(\"../\", \"results/QR\")\n", + "mkpath(baseDir);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "356e3edd", + "metadata": {}, + "outputs": [], + "source": [ + "### time with respect to lambda\n", + "\n", + "# parameters\n", + "lambdas = 10. .^(-15:6)\n", + "epss = [nothing]\n", + "m = 300\n", + "n = 20\n", + "\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"statisticsQR-lambda-m\" * string(m) * \"n\" * string(n) * \"--time.csv\");\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :meantime => Array{Float64}(undef, 0),\n", + " :stdtime => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "# computation\n", + "for (λ, ϵ) ∈ Iterators.product(lambdas, epss) |> collect\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + "\n", + " t = @benchmark begin\n", + " QR = qrfact($gf[:X_hat])\n", + " w = QR \\ $gf[:y_hat]\n", + " end\n", + "\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:meantime], mean(t.times))\n", + " push!(accData[:stdtime], std(t.times))\n", + " println(\"Done: λ \" * string(λ))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b3d71f8-2f4f-44f0-8c97-dd9d78706163", + "metadata": {}, + "outputs": [], + "source": [ + "### error with respect to lambda\n", + "\n", + "# parameters\n", + "lambdas = 10. .^(-15:6)\n", + "epss = [nothing]\n", + "m = 300\n", + "n = 20\n", + "\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"statisticsQR-lambda-m\" * string(m) * \"n\" * string(n) * \"--error.csv\");\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :relative => Array{Float64}(undef, 0),\n", + " :residual => Array{Float64}(undef, 0),\n", + " :stability => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "# computation\n", + "for (λ, ϵ) ∈ Iterators.product(lambdas, epss) |> collect\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + "\n", + " QR = qrfact(gf[:X_hat])\n", + " w = QR \\ gf[:y_hat]\n", + " \n", + " relative_error = norm(w - gf[:w_star]) / norm(gf[:w_star])\n", + " residual = norm(gf[:X_hat] * w - gf[:y_hat]) / norm(gf[:y_hat])\n", + " stability = norm(Q_times_R - gf[:X_hat]) / norm(gf[:X_hat])\n", + "\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:relative], relative_error)\n", + " push!(accData[:residual], residual)\n", + " push!(accData[:stability], stability)\n", + " println(\"Done: λ \" * string(λ))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "387520b7", + "metadata": {}, + "outputs": [], + "source": [ + "### error with respect to lambda\n", + "\n", + "# parameters\n", + "lambdas = 10. .^(-15:6)\n", + "epss = [nothing]\n", + "m = 300\n", + "n = 20\n", + "perturbation = 1e-10\n", + "\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"statisticsQR-forward-m\" * string(m) * \"n\" * string(n) * \"--error.csv\");\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :forwardQ => Array{Float64}(undef, 0),\n", + " :forwardR => Array{Float64}(undef, 0),\n", + " )\n", + "\n", + "# computation\n", + "for (λ, ϵ) ∈ Iterators.product(lambdas, epss) |> collect\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + "\n", + " QR = qrfact(gf[:X_hat])\n", + " w = QR \\ gf[:y_hat]\n", + "\n", + " X_hat_p = gf[:X_hat] + perturbation * randn(size(gf[:X_hat]))\n", + " cond(gf[:X_hat]) |> display\n", + " # cond(X_hat_p) |> display\n", + " QR_p = qrfact(X_hat_p)\n", + "\n", + " forwardQ = norm(QR.Q - QR_p.Q)\n", + " forwardR = norm(QR.R - QR_p.R) / norm(QR.R)\n", + "\n", + " push!(accData[:lambda], λ)\n", + " push!(accData[:forwardQ], forwardQ)\n", + " push!(accData[:forwardR], forwardR)\n", + "\n", + " println(\"Done: λ \" * string(λ))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.9.3", + "language": "julia", + "name": "julia-1.9" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.9.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/project/testing/comparison.ipynb b/project/testing/comparison.ipynb new file mode 100644 index 0000000..55e5d7b --- /dev/null +++ b/project/testing/comparison.ipynb @@ -0,0 +1,674 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "include(\"../QR/housQR.jl\")\n", + "include(\"../utilities/genFunc.jl\")\n", + "include(\"../L-BFGS/OracleFunction.jl\")\n", + "include(\"../L-BFGS/LBFGS.jl\")\n", + "include(\"../L-BFGS/BFGS.jl\")\n", + "include(\"../L-BFGS/SR1.jl\")\n", + "using .LBFGS\n", + "using .BFGS\n", + "using .SR1\n", + "using .OracleFunction\n", + "using .housQR\n", + "using LinearAlgebra, BenchmarkTools, CSV, DataFrames\n", + "\n", + "baseDir = \"results/comparison\";\n", + "mkpath(baseDir);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# QR vs LBFGS running time comparison with respect to n, fixing m, on ill-conditioned matrix\n", + "λ = 1e-12\n", + "m = 200\n", + "runs = 10\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"QRvsLBFGS-n-m\" * string(m) * \"-illcond--time.csv\");\n", + "accData = Dict(\n", + " :n => Array{Float64}(undef, 0),\n", + " :meantimeLBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeLBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsLBFGS => Array{Float64}(undef, 0),\n", + " :meantimeQR => Array{Float64}(undef, 0),\n", + " :stdtimeQR => Array{Float64}(undef, 0),\n", + " :meanallocsQR => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "for n ∈ (0:runs) .* 500 .+ 500\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + "\n", + " t_qr = @benchmark qrfact($gf[:X_hat]) \\ $gf[:y_hat] samples=10 evals=1\n", + "\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " t_lbfgs = @benchmark LimitedMemoryBFGS($ls) samples=10 evals=1\n", + " \n", + " push!(accData[:n], n)\n", + " push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n", + " push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n", + " push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n", + " push!(accData[:meantimeQR], mean(t_qr.times))\n", + " push!(accData[:stdtimeQR], std(t_qr.times))\n", + " push!(accData[:meanallocsQR], mean(t_qr.memory))\n", + " println(\"Done: n \" * string(n))\n", + " flush(stdout)\n", + "end\n", + "\n", + "# CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# QR vs LBFGS running time comparison with respect to n, fixing m, on well-conditioned matrix\n", + "λ = 1e-4\n", + "m = 200\n", + "runs = 10\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"QRvsLBFGS-n-m\" * string(m) * \"-wellcond--time.csv\");\n", + "accData = Dict(\n", + " :n => Array{Float64}(undef, 0),\n", + " :meantimeLBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeLBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsLBFGS => Array{Float64}(undef, 0),\n", + " :meantimeQR => Array{Float64}(undef, 0),\n", + " :stdtimeQR => Array{Float64}(undef, 0),\n", + " :meanallocsQR => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "for n ∈ (0:runs) .* 500 .+ 500\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + "\n", + " t_qr = @benchmark qrfact($gf[:X_hat]) \\ $gf[:y_hat] samples=10 evals=1\n", + "\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " t_lbfgs = @benchmark LimitedMemoryBFGS($ls) samples=10 evals=1\n", + " \n", + " push!(accData[:n], n)\n", + " push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n", + " push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n", + " push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n", + " push!(accData[:meantimeQR], mean(t_qr.times))\n", + " push!(accData[:stdtimeQR], std(t_qr.times))\n", + " push!(accData[:meanallocsQR], mean(t_qr.memory))\n", + " println(\"Done: n \" * string(n))\n", + " flush(stdout)\n", + "end\n", + "\n", + "# CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done: n 500\n", + "Done: n 1000\n", + "Done: n 1500\n", + "Done: n 2000\n", + "Done: n 2500\n", + "Done: n 3000\n", + "Done: n 3500\n", + "Done: n 4000\n", + "Done: n 4500\n", + "Done: n 5000\n", + "Done: n 5500\n" + ] + } + ], + "source": [ + "# BFGS vs LBFGS running time comparison with respect to n, fixing m, on well-conditioned matrix\n", + "λ = 1e-4\n", + "m = 200\n", + "runs = 10\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"BFGSvsLBFGS-n-m\" * string(m) * \"--time.csv\");\n", + "accData = Dict(\n", + " :n => Array{Float64}(undef, 0),\n", + " :meantimeLBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeLBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsLBFGS => Array{Float64}(undef, 0),\n", + " :meantimeBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsBFGS => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "for i ∈ 0:runs\n", + " n = 500 + i * 500\n", + "\n", + "\n", + " t_lbfgs = @benchmark LimitedMemoryBFGS(ls) setup=(ls = (genFunc(:randDataset, λ=$λ, m=$m, n=$n) |> LeastSquaresF)) samples=10 evals=1\n", + " t_bfgs = @benchmark BroydenFletcherGoldfarbShanno(ls) setup=(ls = (genFunc(:randDataset, λ=$λ, m=$m, n=$n) |> LeastSquaresF)) samples=10 evals=1\n", + "\n", + " push!(accData[:n], n)\n", + " push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n", + " push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n", + " push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n", + " push!(accData[:meantimeBFGS], mean(t_bfgs.times))\n", + " push!(accData[:stdtimeBFGS], std(t_bfgs.times))\n", + " push!(accData[:meanallocsBFGS], mean(t_bfgs.memory))\n", + " println(\"Done: n \" * string(n))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done: m 500\n", + "Done: m 1000\n", + "Done: m 1500\n", + "Done: m 2000\n", + "Done: m 2500\n", + "Done: m 3000\n", + "Done: m 3500\n", + "Done: m 4000\n", + "Done: m 4500\n", + "Done: m 5000\n", + "Done: m 5500\n" + ] + } + ], + "source": [ + "# BFGS vs LBFGS running time comparison with respect to m, fixing n, on well-conditioned matrix\n", + "λ = 1e-4\n", + "n = 50\n", + "runs = 10\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"BFGSvsLBFGS-m-n\" * string(n) * \"--time.csv\");\n", + "accData = Dict(\n", + " :m => Array{Float64}(undef, 0),\n", + " :meantimeLBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeLBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsLBFGS => Array{Float64}(undef, 0),\n", + " :meantimeBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsBFGS => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "for i ∈ 0:runs\n", + " m = 500 + i * 500\n", + "\n", + "\n", + " t_lbfgs = @benchmark LimitedMemoryBFGS(ls) setup=(ls = (genFunc(:randDataset, λ=$λ, m=$m, n=$n) |> LeastSquaresF)) samples=10 evals=1\n", + " t_bfgs = @benchmark BroydenFletcherGoldfarbShanno(ls) setup=(ls = (genFunc(:randDataset, λ=$λ, m=$m, n=$n) |> LeastSquaresF)) samples=10 evals=1\n", + "\n", + " push!(accData[:m], m)\n", + " push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n", + " push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n", + " push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n", + " push!(accData[:meantimeBFGS], mean(t_bfgs.times))\n", + " push!(accData[:stdtimeBFGS], std(t_bfgs.times))\n", + " push!(accData[:meanallocsBFGS], mean(t_bfgs.memory))\n", + " println(\"Done: m \" * string(m))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# QR vs LBFGS running time comparison with respect to m, fixing n, on well-conditioned matrix\n", + "λ = 1e-4\n", + "n = 50\n", + "runs = 10\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"QRvsLBFGS-m-n\" * string(n) * \"-wellcond--time.csv\");\n", + "accData = Dict(\n", + " :m => Array{Float64}(undef, 0),\n", + " :meantimeLBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeLBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsLBFGS => Array{Float64}(undef, 0),\n", + " :meantimeQR => Array{Float64}(undef, 0),\n", + " :stdtimeQR => Array{Float64}(undef, 0),\n", + " :meanallocsQR => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "# warm up\n", + "qrfact([0. 1; 1 0]) \\ [1.; 1]\n", + "\n", + "for i ∈ 0:runs\n", + " m = 500 + i * 500\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + "\n", + " t_qr = @benchmark begin\n", + " QR = qrfact($gf[:X_hat])\n", + " w = QR \\ $gf[:y_hat]\n", + " end samples=10 evals=1\n", + "\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " t_lbfgs = @benchmark begin\n", + " LimitedMemoryBFGS($ls)\n", + " end samples=10 evals=1\n", + " \n", + " push!(accData[:m], m)\n", + " push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n", + " push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n", + " push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n", + " push!(accData[:meantimeQR], mean(t_qr.times))\n", + " push!(accData[:stdtimeQR], std(t_qr.times))\n", + " push!(accData[:meanallocsQR], mean(t_qr.memory))\n", + " println(\"Done: m \" * string(m))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# QR vs LBFGS running time comparison with respect to m, fixing n, on ill-conditioned matrix\n", + "λ = 1e-12\n", + "n = 50\n", + "runs = 10\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"QRvsLBFGS-m-n\" * string(n) * \"-illcond--time.csv\");\n", + "accData = Dict(\n", + " :m => Array{Float64}(undef, 0),\n", + " :meantimeLBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeLBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsLBFGS => Array{Float64}(undef, 0),\n", + " :meantimeQR => Array{Float64}(undef, 0),\n", + " :stdtimeQR => Array{Float64}(undef, 0),\n", + " :meanallocsQR => Array{Float64}(undef, 0)\n", + " )\n", + "\n", + "# warm up\n", + "qrfact([0. 1; 1 0]) \\ [1.; 1]\n", + "\n", + "for i ∈ 0:runs\n", + " m = 500 + i * 500\n", + " gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n", + "\n", + " t_qr = @benchmark begin\n", + " QR = qrfact($gf[:X_hat])\n", + " w = QR \\ $gf[:y_hat]\n", + " end samples=10 evals=1\n", + "\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " t_lbfgs = @benchmark begin\n", + " LimitedMemoryBFGS($ls)\n", + " end samples=10 evals=1\n", + " \n", + " push!(accData[:m], m)\n", + " push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n", + " push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n", + " push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n", + " push!(accData[:meantimeQR], mean(t_qr.times))\n", + " push!(accData[:stdtimeQR], std(t_qr.times))\n", + " push!(accData[:meanallocsQR], mean(t_qr.memory))\n", + " println(\"Done: m \" * string(m))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Comparison of all quasi newton methods" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done: m 500\n", + "Done: m 1000\n", + "Done: m 1500\n", + "Done: m 2000\n", + "Done: m 2500\n", + "Done: m 3000\n", + "Done: m 3500\n", + "Done: m 4000\n", + "Done: m 4500\n", + "Done: m 5000\n", + "Done: m 5500\n" + ] + } + ], + "source": [ + "# Quasi newton methods running time comparison, fixing n and varying m, on ill-conditioned matrix\n", + "λ = 1e-12\n", + "n = 50\n", + "runs = 10\n", + "samples = 10\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"Quasi-Newton-Comparison-time-illcond.csv\");\n", + "accData = Dict(\n", + " :m => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeLBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeLBFGS => Array{Float64}(undef, 0),\n", + " :accuracyLBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsLBFGS => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeBFGS => Array{Float64}(undef, 0),\n", + " :accuracyBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsBFGS => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeBFGSDogleg => Array{Float64}(undef, 0),\n", + " :stdtimeBFGSDogleg => Array{Float64}(undef, 0),\n", + " :accuracyBFGSDogleg => Array{Float64}(undef, 0),\n", + " :meanallocsBFGSDogleg => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeDFP => Array{Float64}(undef, 0),\n", + " :stdtimeDFP => Array{Float64}(undef, 0),\n", + " :accuracyDFP => Array{Float64}(undef, 0),\n", + " :meanallocsDFP => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeDFPDogleg => Array{Float64}(undef, 0),\n", + " :stdtimeDFPDogleg => Array{Float64}(undef, 0),\n", + " :accuracyDFPDogleg => Array{Float64}(undef, 0),\n", + " :meanallocsDFPDogleg => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeSR1 => Array{Float64}(undef, 0),\n", + " :stdtimeSR1 => Array{Float64}(undef, 0),\n", + " :accuracySR1 => Array{Float64}(undef, 0),\n", + " :meanallocsSR1 => Array{Float64}(undef, 0)\n", + " \n", + " )\n", + "\n", + "\n", + "for i ∈ 0:runs\n", + " m = 500 + i * 500\n", + "\n", + " gf = genFunc(:randDataset, λ=λ, m=m, n=n)\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " t_lbfgs = @benchmark begin\n", + " LimitedMemoryBFGS($ls)\n", + " end samples=samples evals=1\n", + " res_lbfgs = LimitedMemoryBFGS(ls)\n", + "\n", + "\n", + " BFGS.BFGSorDFP = :BFGS\n", + "\n", + " t_bfgs = @benchmark begin\n", + " BroydenFletcherGoldfarbShanno($ls)\n", + " end samples=samples evals=1\n", + " res_bfgs = BroydenFletcherGoldfarbShanno(ls)\n", + "\n", + " t_bfgs_dogleg = @benchmark begin\n", + " BroydenFletcherGoldfarbShannoDogleg($ls)\n", + " end samples=samples evals=1\n", + " res_bfgs_dogleg = BroydenFletcherGoldfarbShannoDogleg(ls)\n", + "\n", + " BFGS.BFGSorDFP = :DFP\n", + "\n", + " t_dfp = @benchmark begin\n", + " BroydenFletcherGoldfarbShanno($ls)\n", + " end samples=samples evals=1\n", + " res_dfp = BroydenFletcherGoldfarbShanno(ls)\n", + "\n", + " t_dfp_dogleg = @benchmark begin\n", + " BroydenFletcherGoldfarbShannoDogleg($ls)\n", + " end samples=samples evals=1\n", + " res_dfp_dogleg = BroydenFletcherGoldfarbShannoDogleg(ls)\n", + "\n", + " t_sr1 = @benchmark begin\n", + " SymmetricRank1($ls)\n", + " end samples=samples evals=1\n", + " res_sr1 = SymmetricRank1(ls)\n", + " \n", + " push!(accData[:m], m)\n", + " push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n", + " push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n", + " push!(accData[:accuracyLBFGS], norm(res_lbfgs.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n", + "\n", + " push!(accData[:meantimeBFGS], mean(t_bfgs.times))\n", + " push!(accData[:stdtimeBFGS], std(t_bfgs.times))\n", + " push!(accData[:accuracyBFGS], norm(res_bfgs.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsBFGS], mean(t_bfgs.memory))\n", + "\n", + " push!(accData[:meantimeBFGSDogleg], mean(t_bfgs_dogleg.times))\n", + " push!(accData[:stdtimeBFGSDogleg], std(t_bfgs_dogleg.times))\n", + " push!(accData[:accuracyBFGSDogleg], norm(res_bfgs_dogleg.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsBFGSDogleg], mean(t_bfgs_dogleg.memory))\n", + "\n", + " push!(accData[:meantimeDFP], mean(t_dfp.times))\n", + " push!(accData[:stdtimeDFP], std(t_dfp.times))\n", + " push!(accData[:accuracyDFP], norm(res_dfp.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsDFP], mean(t_dfp.memory))\n", + "\n", + " push!(accData[:meantimeDFPDogleg], mean(t_dfp_dogleg.times))\n", + " push!(accData[:stdtimeDFPDogleg], std(t_dfp_dogleg.times))\n", + " push!(accData[:accuracyDFPDogleg], norm(res_dfp_dogleg.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsDFPDogleg], mean(t_dfp_dogleg.memory))\n", + "\n", + " push!(accData[:meantimeSR1], mean(t_sr1.times))\n", + " push!(accData[:stdtimeSR1], std(t_sr1.times))\n", + " push!(accData[:accuracySR1], norm(res_sr1.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsSR1], mean(t_sr1.memory))\n", + " println(\"Done: m \" * string(m))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done: m 500\n", + "Done: m 1000\n", + "Done: m 1500\n", + "Done: m 2000\n", + "Done: m 2500\n", + "Done: m 3000\n", + "Done: m 3500\n", + "Done: m 4000\n", + "Done: m 4500\n", + "Done: m 5000\n", + "Done: m 5500\n" + ] + } + ], + "source": [ + "# Quasi newton methods running time comparison, fixing n and varying m, on well-conditioned matrix\n", + "λ = 1e-4\n", + "n = 50\n", + "runs = 10\n", + "samples = 10\n", + "\n", + "# output csv\n", + "outputvsc = joinpath(baseDir, \"Quasi-Newton-Comparison-time-wellcond.csv\");\n", + "accData = Dict(\n", + " :m => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeLBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeLBFGS => Array{Float64}(undef, 0),\n", + " :accuracyLBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsLBFGS => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeBFGS => Array{Float64}(undef, 0),\n", + " :stdtimeBFGS => Array{Float64}(undef, 0),\n", + " :accuracyBFGS => Array{Float64}(undef, 0),\n", + " :meanallocsBFGS => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeBFGSDogleg => Array{Float64}(undef, 0),\n", + " :stdtimeBFGSDogleg => Array{Float64}(undef, 0),\n", + " :accuracyBFGSDogleg => Array{Float64}(undef, 0),\n", + " :meanallocsBFGSDogleg => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeDFP => Array{Float64}(undef, 0),\n", + " :stdtimeDFP => Array{Float64}(undef, 0),\n", + " :accuracyDFP => Array{Float64}(undef, 0),\n", + " :meanallocsDFP => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeDFPDogleg => Array{Float64}(undef, 0),\n", + " :stdtimeDFPDogleg => Array{Float64}(undef, 0),\n", + " :accuracyDFPDogleg => Array{Float64}(undef, 0),\n", + " :meanallocsDFPDogleg => Array{Float64}(undef, 0),\n", + "\n", + " :meantimeSR1 => Array{Float64}(undef, 0),\n", + " :stdtimeSR1 => Array{Float64}(undef, 0),\n", + " :accuracySR1 => Array{Float64}(undef, 0),\n", + " :meanallocsSR1 => Array{Float64}(undef, 0)\n", + " \n", + " )\n", + "\n", + "\n", + "for i ∈ 0:runs\n", + " m = 500 + i * 500\n", + "\n", + " gf = genFunc(:randDataset, λ=λ, m=m, n=n)\n", + " ls = LeastSquaresF(gf)\n", + "\n", + " t_lbfgs = @benchmark begin\n", + " LimitedMemoryBFGS($ls)\n", + " end samples=samples evals=1\n", + " res_lbfgs = LimitedMemoryBFGS(ls)\n", + "\n", + "\n", + " BFGS.BFGSorDFP = :BFGS\n", + "\n", + " t_bfgs = @benchmark begin\n", + " BroydenFletcherGoldfarbShanno($ls)\n", + " end samples=samples evals=1\n", + " res_bfgs = BroydenFletcherGoldfarbShanno(ls)\n", + "\n", + " t_bfgs_dogleg = @benchmark begin\n", + " BroydenFletcherGoldfarbShannoDogleg($ls)\n", + " end samples=samples evals=1\n", + " res_bfgs_dogleg = BroydenFletcherGoldfarbShannoDogleg(ls)\n", + "\n", + " BFGS.BFGSorDFP = :DFP\n", + "\n", + " t_dfp = @benchmark begin\n", + " BroydenFletcherGoldfarbShanno($ls)\n", + " end samples=samples evals=1\n", + " res_dfp = BroydenFletcherGoldfarbShanno(ls)\n", + "\n", + " t_dfp_dogleg = @benchmark begin\n", + " BroydenFletcherGoldfarbShannoDogleg($ls)\n", + " end samples=samples evals=1\n", + " res_dfp_dogleg = BroydenFletcherGoldfarbShannoDogleg(ls)\n", + "\n", + " t_sr1 = @benchmark begin\n", + " SymmetricRank1($ls)\n", + " end samples=samples evals=1\n", + " res_sr1 = SymmetricRank1(ls)\n", + " \n", + " push!(accData[:m], m)\n", + " push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n", + " push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n", + " push!(accData[:accuracyLBFGS], norm(res_lbfgs.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n", + "\n", + " push!(accData[:meantimeBFGS], mean(t_bfgs.times))\n", + " push!(accData[:stdtimeBFGS], std(t_bfgs.times))\n", + " push!(accData[:accuracyBFGS], norm(res_bfgs.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsBFGS], mean(t_bfgs.memory))\n", + "\n", + " push!(accData[:meantimeBFGSDogleg], mean(t_bfgs_dogleg.times))\n", + " push!(accData[:stdtimeBFGSDogleg], std(t_bfgs_dogleg.times))\n", + " push!(accData[:accuracyBFGSDogleg], norm(res_bfgs_dogleg.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsBFGSDogleg], mean(t_bfgs_dogleg.memory))\n", + "\n", + " push!(accData[:meantimeDFP], mean(t_dfp.times))\n", + " push!(accData[:stdtimeDFP], std(t_dfp.times))\n", + " push!(accData[:accuracyDFP], norm(res_dfp.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsDFP], mean(t_dfp.memory))\n", + "\n", + " push!(accData[:meantimeDFPDogleg], mean(t_dfp_dogleg.times))\n", + " push!(accData[:stdtimeDFPDogleg], std(t_dfp_dogleg.times))\n", + " push!(accData[:accuracyDFPDogleg], norm(res_dfp_dogleg.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsDFPDogleg], mean(t_dfp_dogleg.memory))\n", + "\n", + " push!(accData[:meantimeSR1], mean(t_sr1.times))\n", + " push!(accData[:stdtimeSR1], std(t_sr1.times))\n", + " push!(accData[:accuracySR1], norm(res_sr1.x - gf[:w_star]) / norm(gf[:w_star]))\n", + " push!(accData[:meanallocsSR1], mean(t_sr1.memory))\n", + " println(\"Done: m \" * string(m))\n", + " flush(stdout)\n", + "end\n", + "\n", + "CSV.write(outputvsc, DataFrame(accData));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.9.3", + "language": "julia", + "name": "julia-1.9" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.9.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/project/testing/conditioning.ipynb b/project/testing/conditioning.ipynb new file mode 100644 index 0000000..fa1b563 --- /dev/null +++ b/project/testing/conditioning.ipynb @@ -0,0 +1,64 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include(\"../../utilities/dataset.jl\")\n", + "using CSV, DataFrames\n", + "\n", + "baseDir = joinpath(\"../\", \"results/conditioning\")\n", + "mkpath(baseDir);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lambdas = 10. .^(-15:6)\n", + "\n", + "outputvsc = joinpath(baseDir, \"conditioning.csv\");\n", + "accData = Dict(\n", + " :lambda => Array{Float64}(undef, 0),\n", + " :condnum => Array{Float64}(undef, 0),\n", + " )\n", + "\n", + "for lambda in lambdas\n", + " X_hat, _, _, _ = get_dataset(\"../data_for_testing/dataset.csv\", lambda)\n", + " condnum = cond(X_hat)\n", + " push!(accData[:lambda], lambda)\n", + " push!(accData[:condnum], condnum)\n", + "end\n", + "\n", + "df = DataFrame(accData)\n", + "# CSV.write(outputvsc, df);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.9.3", + "language": "julia", + "name": "julia-1.9" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.9.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/project/thinQR.jl b/project/thinQR.jl deleted file mode 100644 index 114b9d6..0000000 --- a/project/thinQR.jl +++ /dev/null @@ -1,120 +0,0 @@ -module thinQR - -import Base: size, show, getproperty, getfield, propertynames, \, * -using LinearAlgebra: norm, I, triu, diagm, ldiv!, dot - -export QRthin, qrhous!, qyhous, qyhoust - -mutable struct QRthin{T <: Real} - A::AbstractVecOrMat{T} - d::AbstractArray{T} - AQ::Union{Nothing, AbstractVecOrMat{T}} - AR::Union{Nothing, AbstractVecOrMat{T}} - - QRthin(A, d, AQ=nothing, AR=nothing) = new{eltype(A)}(A, d, AQ, AR) -end - -function qrhous!(A::AbstractMatrix{T})::QRthin{T} where T - m, n = size(A) - d = zeros(n) - @inbounds begin - for j ∈ 1:n - s = norm(A[j:m, j]) - # iszero(s) && throw(ArgumentError("The matrix A is singular")) - d[j]= copysign(s, -A[j,j]) - fak = sqrt(s * (s + abs(A[j,j]))) - A[j,j] -= d[j] - - A[j:m, j] ./= fak - - if j < n - A[j:m, j+1:n] -= A[j:m, j] * (A[j:m, j]' * A[j:m, j+1:n]) - #for k ∈ j+1:n - # A[j:m, k] -= A[j:m, j] .* dot(A[j:m, j], A[j:m, k]) - #end - end - end - end - - return QRthin(A, d) -end - -function qyhous(A::QRthin{T}, y::AbstractArray{T}) where T - m, n = size(A.A) - z = deepcopy(y) - for j ∈ n:-1:1 - # z[j:m] = z[j:m] - A.A[j:m, j] * (A.A[j:m, j]' * z[j:m]) - z[j:m] -= A.A[j:m, j] .* dot(A.A[j:m, j], z[j:m]) - end - return z -end - -function qyhoust(A::QRthin{T}, y::AbstractArray{T}) where T - m, n = size(A.A) - z = deepcopy(y) - for j ∈ 1:n - # z[j:m] = z[j:m] - A.A[j:m, j] * (A.A[j:m, j]' * z[j:m]) - z[j:m] -= A.A[j:m, j] .* dot(A.A[j:m, j], z[j:m]) - end - return z -end - -function calculateQ(A::QRthin{T}) where T - if A.AQ != nothing - return A.AQ - end - m, n = size(A.A) - A.AQ = zeros(m, 0) - id = Matrix{eltype(A.A)}(I, m, n) - for i ∈ eachcol(id) - A.AQ = [A.AQ qyhous(A, i)] - end - return A.AQ -end - -function calculateR(A::QRthin{T}) where T - if A.AR != nothing - return A.AR - end - m, n = size(A.A) - A.AR = vcat(triu(A.A[1:n, :], 1) + diagm(A.d), zeros(m-n, n)) - return A.AR -end - -function Base.show(io::IO, mime::MIME{Symbol("text/plain")}, A::QRthin{T}) where T - summary(io, A); println(io) - print(io, "Q factor: ") - show(io, mime, A.Q) - print(io, "\nR factor: ") - show(io, mime, A.R) -end - -function Base.getproperty(A::QRthin{T}, d::Symbol) where T - if d === :R - return calculateR(A) - elseif d === :Q - return calculateQ(A) - else - getfield(A, d) - end -end - -Base.propertynames(A::QRthin, private::Bool=false) = (:R, :Q, (private ? fieldnames(typeof(A)) : ())...) - -Base.size(A::QRthin) = size(getfield(A, :A)) - -function (\)(A::QRthin{T}, b::AbstractVector{T}) where T - n, m = size(A) - v = qyhoust(A, b) - x = zeros(m) - for j ∈ m:-1:1 - x[j] = (v[j] - dot(x[j+1:m], A.A[j, j+1:m])) * A.d[j]^-1 - end - return x -end - -function (*)(A::QRthin{T}, x::AbstractVecOrMat{T}) where T - return qyhous(A, (A.R * x)) -end - -end # module \ No newline at end of file diff --git a/project/utilities/dataset.jl b/project/utilities/dataset.jl new file mode 100644 index 0000000..aff6b3b --- /dev/null +++ b/project/utilities/dataset.jl @@ -0,0 +1,42 @@ +using CSV +using DataFrames +using LinearAlgebra + +@doc raw""" + get_dataset(file_path::String, lambda::Float64) + +Reads the ML-CUP-23/24 dataset from the given file path and returns the +necessary matrices and vectors to solve the problem. The LS problem is +solved with the standard solver in Julia. + +### Input + +- 'file_path' -- the path to the dataset file. +- 'lambda' -- the regularization parameter. + +### Output + +A tuple containing + +- 'X_hat' -- the augmented matrix X. +- 'y_hat' -- the augmented vector y. +- 'start' -- the initial starting point. +- 'w_star' -- the optimal solution. + +""" +function get_dataset(filePath::String, λ::Real) + X = CSV.File(filePath; header=false) |> DataFrame + X = Matrix(X) + + m, n = size(X) + X_hat = [transpose(X); λ .* I(m)] + y_hat = [(rand(n).*2 .-1); zeros(m)] + + # initial starting point + start = ones(m) + + # w_star is the optimal solution provided by the julia solver + w_star = X_hat \ y_hat + + return X_hat, y_hat, start, w_star +end diff --git a/project/utilities/genFunc.jl b/project/utilities/genFunc.jl new file mode 100644 index 0000000..ab50193 --- /dev/null +++ b/project/utilities/genFunc.jl @@ -0,0 +1,91 @@ +include("dataset.jl") +using Random + +datasetpath = joinpath(@__DIR__, "../data_for_testing/dataset.csv") +VALID = [:dataset, :randDataset, :exactRandDataset, :rand, :exactRand] + +@doc raw""" +genFunc(t::Symbol, [lambda::Real=1, m::Integer=1000, n::Integer=10, rng=default_rng()]) + +Generates different matrices for the least squares problem + +### Input + +- `t` -- The type of problem to generate. Possible values are: +`:dataset`, +`:randDataset`, +`:exactRandDataset`, +`:rand`, +`:exactRand` + +### Output + +A named tuple with the matrix ``\hat{X}``, the vector ``\hat{y}``, the starting vector and the solution vector ``w^*``. + +See also [`gen_dataset`](@ref). +""" +function genFunc( + t::Symbol; + λ::Real=1, + m::Integer=1000, + n::Integer=10, + rng=Random.default_rng())::NamedTuple + if t ∉ VALID + throw(ArgumentError("The type of matrix to produce is not recognized, available types: " * join(String.(VALID), ", "))) + end + + if t == :dataset + tmp = get_dataset(datasetpath, λ) + return (; :X_hat => tmp[1], :y_hat => tmp[2], :start => tmp[3], :w_star => tmp[4]) + end + + if t == :randDataset + X_hat = [(rand(rng, n, m).*2 .-1); λ .* I(m)] + y_hat = [(rand(rng, n).*2 .-1); zeros(m)] + + # initial starting point + start = ones(m) + + # w_star is the optimal solution of the julia solver + w_star = X_hat \ y_hat + + return (; :X_hat => X_hat, :y_hat => y_hat, :start => start, :w_star => w_star) + end + + if t == :exactRandDataset + X_hat = [(rand(rng, n, m).*2 .-1); λ .* I(m)] + w_star = rand(rng, m).*2 .-1 + + # initial starting point + start = ones(m) + + y_hat = X_hat * w_star + + return (; :X_hat => X_hat, :y_hat => y_hat, :start => start, :w_star => w_star) + end + + if t == :rand + X_hat = rand(rng, m+n, m).*2 .-1 + y_hat = rand(rng, m+n) + + # initial starting point + start = ones(m) + + # w_star is the optimal solution of the julia solver + w_star = X_hat \ y_hat + + return (; :X_hat => X_hat, :y_hat => y_hat, :start => start, :w_star => w_star) + end + + if t == :exactRand + X_hat = rand(rng, m+n, m).*2 .-1 + w_star = rand(rng, m).*2 .-1 + + # initial starting point + start = ones(m) + + y_hat = X_hat * w_star + + return (; :X_hat => X_hat, :y_hat => y_hat, :start => start, :w_star => w_star) + end +end