{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "include(\"../QR/housQR.jl\")\n",
    "include(\"../utilities/genFunc.jl\")\n",
    "include(\"../L-BFGS/OracleFunction.jl\")\n",
    "include(\"../L-BFGS/LBFGS.jl\")\n",
    "include(\"../L-BFGS/BFGS.jl\")\n",
    "include(\"../L-BFGS/SR1.jl\")\n",
    "using .LBFGS\n",
    "using .BFGS\n",
    "using .SR1\n",
    "using .OracleFunction\n",
    "using .housQR\n",
    "using LinearAlgebra, BenchmarkTools, CSV, DataFrames\n",
    "\n",
    "baseDir = \"results/comparison\";\n",
    "mkpath(baseDir);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# QR vs LBFGS running time comparison with respect to n, fixing m, on ill-conditioned matrix\n",
    "λ    = 1e-12\n",
    "m    = 200\n",
    "runs = 10\n",
    "\n",
    "# output csv\n",
    "outputvsc = joinpath(baseDir, \"QRvsLBFGS-n-m\" * string(m) * \"-illcond--time.csv\");\n",
    "accData = Dict(\n",
    "    :n   => Array{Float64}(undef, 0),\n",
    "    :meantimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsLBFGS => Array{Float64}(undef, 0),\n",
    "    :meantimeQR => Array{Float64}(undef, 0),\n",
    "    :stdtimeQR => Array{Float64}(undef, 0),\n",
    "    :meanallocsQR => Array{Float64}(undef, 0)\n",
    "    )\n",
    "\n",
    "for n ∈ (0:runs) .* 500 .+ 500\n",
    "    gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n",
    "\n",
    "    t_qr = @benchmark qrfact($gf[:X_hat]) \\ $gf[:y_hat] samples=10 evals=1\n",
    "\n",
    "    ls = LeastSquaresF(gf)\n",
    "\n",
    "    t_lbfgs = @benchmark LimitedMemoryBFGS($ls) samples=10 evals=1\n",
    "    \n",
    "    push!(accData[:n], n)\n",
    "    push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n",
    "    push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n",
    "    push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n",
    "    push!(accData[:meantimeQR], mean(t_qr.times))\n",
    "    push!(accData[:stdtimeQR], std(t_qr.times))\n",
    "    push!(accData[:meanallocsQR], mean(t_qr.memory))\n",
    "    println(\"Done: n \" * string(n))\n",
    "    flush(stdout)\n",
    "end\n",
    "\n",
    "# CSV.write(outputvsc, DataFrame(accData));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# QR vs LBFGS running time comparison with respect to n, fixing m, on well-conditioned matrix\n",
    "λ    = 1e-4\n",
    "m    = 200\n",
    "runs = 10\n",
    "\n",
    "# output csv\n",
    "outputvsc = joinpath(baseDir, \"QRvsLBFGS-n-m\" * string(m) * \"-wellcond--time.csv\");\n",
    "accData = Dict(\n",
    "    :n   => Array{Float64}(undef, 0),\n",
    "    :meantimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsLBFGS => Array{Float64}(undef, 0),\n",
    "    :meantimeQR => Array{Float64}(undef, 0),\n",
    "    :stdtimeQR => Array{Float64}(undef, 0),\n",
    "    :meanallocsQR => Array{Float64}(undef, 0)\n",
    "    )\n",
    "\n",
    "for n ∈ (0:runs) .* 500 .+ 500\n",
    "    gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n",
    "\n",
    "    t_qr = @benchmark qrfact($gf[:X_hat]) \\ $gf[:y_hat] samples=10 evals=1\n",
    "\n",
    "    ls = LeastSquaresF(gf)\n",
    "\n",
    "    t_lbfgs = @benchmark LimitedMemoryBFGS($ls) samples=10 evals=1\n",
    "    \n",
    "    push!(accData[:n], n)\n",
    "    push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n",
    "    push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n",
    "    push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n",
    "    push!(accData[:meantimeQR], mean(t_qr.times))\n",
    "    push!(accData[:stdtimeQR], std(t_qr.times))\n",
    "    push!(accData[:meanallocsQR], mean(t_qr.memory))\n",
    "    println(\"Done: n \" * string(n))\n",
    "    flush(stdout)\n",
    "end\n",
    "\n",
    "# CSV.write(outputvsc, DataFrame(accData));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done: n 500\n",
      "Done: n 1000\n",
      "Done: n 1500\n",
      "Done: n 2000\n",
      "Done: n 2500\n",
      "Done: n 3000\n",
      "Done: n 3500\n",
      "Done: n 4000\n",
      "Done: n 4500\n",
      "Done: n 5000\n",
      "Done: n 5500\n"
     ]
    }
   ],
   "source": [
    "# BFGS vs LBFGS running time comparison with respect to n, fixing m, on well-conditioned matrix\n",
    "λ    = 1e-4\n",
    "m    = 200\n",
    "runs = 10\n",
    "\n",
    "# output csv\n",
    "outputvsc = joinpath(baseDir, \"BFGSvsLBFGS-n-m\" * string(m) * \"--time.csv\");\n",
    "accData = Dict(\n",
    "    :n   => Array{Float64}(undef, 0),\n",
    "    :meantimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsLBFGS => Array{Float64}(undef, 0),\n",
    "    :meantimeBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsBFGS => Array{Float64}(undef, 0)\n",
    "    )\n",
    "\n",
    "for i ∈ 0:runs\n",
    "    n = 500 + i * 500\n",
    "\n",
    "\n",
    "    t_lbfgs = @benchmark LimitedMemoryBFGS(ls) setup=(ls = (genFunc(:randDataset, λ=$λ, m=$m, n=$n) |> LeastSquaresF)) samples=10 evals=1\n",
    "    t_bfgs = @benchmark BroydenFletcherGoldfarbShanno(ls) setup=(ls = (genFunc(:randDataset, λ=$λ, m=$m, n=$n) |> LeastSquaresF)) samples=10 evals=1\n",
    "\n",
    "    push!(accData[:n], n)\n",
    "    push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n",
    "    push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n",
    "    push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n",
    "    push!(accData[:meantimeBFGS], mean(t_bfgs.times))\n",
    "    push!(accData[:stdtimeBFGS], std(t_bfgs.times))\n",
    "    push!(accData[:meanallocsBFGS], mean(t_bfgs.memory))\n",
    "    println(\"Done: n \" * string(n))\n",
    "    flush(stdout)\n",
    "end\n",
    "\n",
    "CSV.write(outputvsc, DataFrame(accData));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done: m 500\n",
      "Done: m 1000\n",
      "Done: m 1500\n",
      "Done: m 2000\n",
      "Done: m 2500\n",
      "Done: m 3000\n",
      "Done: m 3500\n",
      "Done: m 4000\n",
      "Done: m 4500\n",
      "Done: m 5000\n",
      "Done: m 5500\n"
     ]
    }
   ],
   "source": [
    "# BFGS vs LBFGS running time comparison with respect to m, fixing n, on well-conditioned matrix\n",
    "λ    = 1e-4\n",
    "n    = 50\n",
    "runs = 10\n",
    "\n",
    "# output csv\n",
    "outputvsc = joinpath(baseDir, \"BFGSvsLBFGS-m-n\" * string(n) * \"--time.csv\");\n",
    "accData = Dict(\n",
    "    :m   => Array{Float64}(undef, 0),\n",
    "    :meantimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsLBFGS => Array{Float64}(undef, 0),\n",
    "    :meantimeBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsBFGS => Array{Float64}(undef, 0)\n",
    "    )\n",
    "\n",
    "for i ∈ 0:runs\n",
    "    m = 500 + i * 500\n",
    "\n",
    "\n",
    "    t_lbfgs = @benchmark LimitedMemoryBFGS(ls) setup=(ls = (genFunc(:randDataset, λ=$λ, m=$m, n=$n) |> LeastSquaresF)) samples=10 evals=1\n",
    "    t_bfgs = @benchmark BroydenFletcherGoldfarbShanno(ls) setup=(ls = (genFunc(:randDataset, λ=$λ, m=$m, n=$n) |> LeastSquaresF)) samples=10 evals=1\n",
    "\n",
    "    push!(accData[:m], m)\n",
    "    push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n",
    "    push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n",
    "    push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n",
    "    push!(accData[:meantimeBFGS], mean(t_bfgs.times))\n",
    "    push!(accData[:stdtimeBFGS], std(t_bfgs.times))\n",
    "    push!(accData[:meanallocsBFGS], mean(t_bfgs.memory))\n",
    "    println(\"Done: m \" * string(m))\n",
    "    flush(stdout)\n",
    "end\n",
    "\n",
    "CSV.write(outputvsc, DataFrame(accData));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# QR vs LBFGS running time comparison with respect to m, fixing n, on well-conditioned matrix\n",
    "λ    = 1e-4\n",
    "n    = 50\n",
    "runs = 10\n",
    "\n",
    "# output csv\n",
    "outputvsc = joinpath(baseDir, \"QRvsLBFGS-m-n\" * string(n) * \"-wellcond--time.csv\");\n",
    "accData = Dict(\n",
    "    :m   => Array{Float64}(undef, 0),\n",
    "    :meantimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsLBFGS => Array{Float64}(undef, 0),\n",
    "    :meantimeQR => Array{Float64}(undef, 0),\n",
    "    :stdtimeQR => Array{Float64}(undef, 0),\n",
    "    :meanallocsQR => Array{Float64}(undef, 0)\n",
    "    )\n",
    "\n",
    "# warm up\n",
    "qrfact([0. 1; 1 0]) \\ [1.; 1]\n",
    "\n",
    "for i ∈ 0:runs\n",
    "    m = 500 + i * 500\n",
    "    gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n",
    "\n",
    "    t_qr = @benchmark begin\n",
    "        QR = qrfact($gf[:X_hat])\n",
    "        w = QR \\ $gf[:y_hat]\n",
    "    end samples=10 evals=1\n",
    "\n",
    "    ls = LeastSquaresF(gf)\n",
    "\n",
    "    t_lbfgs = @benchmark begin\n",
    "        LimitedMemoryBFGS($ls)\n",
    "    end samples=10 evals=1\n",
    "    \n",
    "    push!(accData[:m], m)\n",
    "    push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n",
    "    push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n",
    "    push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n",
    "    push!(accData[:meantimeQR], mean(t_qr.times))\n",
    "    push!(accData[:stdtimeQR], std(t_qr.times))\n",
    "    push!(accData[:meanallocsQR], mean(t_qr.memory))\n",
    "    println(\"Done: m \" * string(m))\n",
    "    flush(stdout)\n",
    "end\n",
    "\n",
    "CSV.write(outputvsc, DataFrame(accData));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# QR vs LBFGS running time comparison with respect to m, fixing n, on ill-conditioned matrix\n",
    "λ    = 1e-12\n",
    "n    = 50\n",
    "runs = 10\n",
    "\n",
    "# output csv\n",
    "outputvsc = joinpath(baseDir, \"QRvsLBFGS-m-n\" * string(n) * \"-illcond--time.csv\");\n",
    "accData = Dict(\n",
    "    :m   => Array{Float64}(undef, 0),\n",
    "    :meantimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsLBFGS => Array{Float64}(undef, 0),\n",
    "    :meantimeQR => Array{Float64}(undef, 0),\n",
    "    :stdtimeQR => Array{Float64}(undef, 0),\n",
    "    :meanallocsQR => Array{Float64}(undef, 0)\n",
    "    )\n",
    "\n",
    "# warm up\n",
    "qrfact([0. 1; 1 0]) \\ [1.; 1]\n",
    "\n",
    "for i ∈ 0:runs\n",
    "    m = 500 + i * 500\n",
    "    gf = genFunc(:exactRandDataset, λ=λ, m=m, n=n)\n",
    "\n",
    "    t_qr = @benchmark begin\n",
    "        QR = qrfact($gf[:X_hat])\n",
    "        w = QR \\ $gf[:y_hat]\n",
    "    end samples=10 evals=1\n",
    "\n",
    "    ls = LeastSquaresF(gf)\n",
    "\n",
    "    t_lbfgs = @benchmark begin\n",
    "        LimitedMemoryBFGS($ls)\n",
    "    end samples=10 evals=1\n",
    "    \n",
    "    push!(accData[:m], m)\n",
    "    push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n",
    "    push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n",
    "    push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n",
    "    push!(accData[:meantimeQR], mean(t_qr.times))\n",
    "    push!(accData[:stdtimeQR], std(t_qr.times))\n",
    "    push!(accData[:meanallocsQR], mean(t_qr.memory))\n",
    "    println(\"Done: m \" * string(m))\n",
    "    flush(stdout)\n",
    "end\n",
    "\n",
    "CSV.write(outputvsc, DataFrame(accData));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparison of all quasi newton methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done: m 500\n",
      "Done: m 1000\n",
      "Done: m 1500\n",
      "Done: m 2000\n",
      "Done: m 2500\n",
      "Done: m 3000\n",
      "Done: m 3500\n",
      "Done: m 4000\n",
      "Done: m 4500\n",
      "Done: m 5000\n",
      "Done: m 5500\n"
     ]
    }
   ],
   "source": [
    "# Quasi newton methods running time comparison, fixing n and varying m, on ill-conditioned matrix\n",
    "λ    = 1e-12\n",
    "n    = 50\n",
    "runs = 10\n",
    "samples = 10\n",
    "\n",
    "# output csv\n",
    "outputvsc = joinpath(baseDir, \"Quasi-Newton-Comparison-time-illcond.csv\");\n",
    "accData = Dict(\n",
    "    :m   => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :accuracyLBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsLBFGS => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeBFGS => Array{Float64}(undef, 0),\n",
    "    :accuracyBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsBFGS => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeBFGSDogleg => Array{Float64}(undef, 0),\n",
    "    :stdtimeBFGSDogleg => Array{Float64}(undef, 0),\n",
    "    :accuracyBFGSDogleg => Array{Float64}(undef, 0),\n",
    "    :meanallocsBFGSDogleg => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeDFP => Array{Float64}(undef, 0),\n",
    "    :stdtimeDFP => Array{Float64}(undef, 0),\n",
    "    :accuracyDFP => Array{Float64}(undef, 0),\n",
    "    :meanallocsDFP => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeDFPDogleg => Array{Float64}(undef, 0),\n",
    "    :stdtimeDFPDogleg => Array{Float64}(undef, 0),\n",
    "    :accuracyDFPDogleg => Array{Float64}(undef, 0),\n",
    "    :meanallocsDFPDogleg => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeSR1 => Array{Float64}(undef, 0),\n",
    "    :stdtimeSR1 => Array{Float64}(undef, 0),\n",
    "    :accuracySR1 => Array{Float64}(undef, 0),\n",
    "    :meanallocsSR1 => Array{Float64}(undef, 0)\n",
    "    \n",
    "    )\n",
    "\n",
    "\n",
    "for i ∈ 0:runs\n",
    "    m = 500 + i * 500\n",
    "\n",
    "    gf = genFunc(:randDataset, λ=λ, m=m, n=n)\n",
    "    ls = LeastSquaresF(gf)\n",
    "\n",
    "    t_lbfgs = @benchmark begin\n",
    "        LimitedMemoryBFGS($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_lbfgs = LimitedMemoryBFGS(ls)\n",
    "\n",
    "\n",
    "    BFGS.BFGSorDFP = :BFGS\n",
    "\n",
    "    t_bfgs = @benchmark begin\n",
    "        BroydenFletcherGoldfarbShanno($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_bfgs = BroydenFletcherGoldfarbShanno(ls)\n",
    "\n",
    "    t_bfgs_dogleg = @benchmark begin\n",
    "        BroydenFletcherGoldfarbShannoDogleg($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_bfgs_dogleg = BroydenFletcherGoldfarbShannoDogleg(ls)\n",
    "\n",
    "    BFGS.BFGSorDFP = :DFP\n",
    "\n",
    "    t_dfp = @benchmark begin\n",
    "        BroydenFletcherGoldfarbShanno($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_dfp = BroydenFletcherGoldfarbShanno(ls)\n",
    "\n",
    "    t_dfp_dogleg = @benchmark begin\n",
    "        BroydenFletcherGoldfarbShannoDogleg($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_dfp_dogleg = BroydenFletcherGoldfarbShannoDogleg(ls)\n",
    "\n",
    "    t_sr1 = @benchmark begin\n",
    "        SymmetricRank1($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_sr1 = SymmetricRank1(ls)\n",
    "    \n",
    "    push!(accData[:m], m)\n",
    "    push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n",
    "    push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n",
    "    push!(accData[:accuracyLBFGS], norm(res_lbfgs.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n",
    "\n",
    "    push!(accData[:meantimeBFGS], mean(t_bfgs.times))\n",
    "    push!(accData[:stdtimeBFGS], std(t_bfgs.times))\n",
    "    push!(accData[:accuracyBFGS], norm(res_bfgs.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsBFGS], mean(t_bfgs.memory))\n",
    "\n",
    "    push!(accData[:meantimeBFGSDogleg], mean(t_bfgs_dogleg.times))\n",
    "    push!(accData[:stdtimeBFGSDogleg], std(t_bfgs_dogleg.times))\n",
    "    push!(accData[:accuracyBFGSDogleg], norm(res_bfgs_dogleg.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsBFGSDogleg], mean(t_bfgs_dogleg.memory))\n",
    "\n",
    "    push!(accData[:meantimeDFP], mean(t_dfp.times))\n",
    "    push!(accData[:stdtimeDFP], std(t_dfp.times))\n",
    "    push!(accData[:accuracyDFP], norm(res_dfp.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsDFP], mean(t_dfp.memory))\n",
    "\n",
    "    push!(accData[:meantimeDFPDogleg], mean(t_dfp_dogleg.times))\n",
    "    push!(accData[:stdtimeDFPDogleg], std(t_dfp_dogleg.times))\n",
    "    push!(accData[:accuracyDFPDogleg], norm(res_dfp_dogleg.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsDFPDogleg], mean(t_dfp_dogleg.memory))\n",
    "\n",
    "    push!(accData[:meantimeSR1], mean(t_sr1.times))\n",
    "    push!(accData[:stdtimeSR1], std(t_sr1.times))\n",
    "    push!(accData[:accuracySR1], norm(res_sr1.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsSR1], mean(t_sr1.memory))\n",
    "    println(\"Done: m \" * string(m))\n",
    "    flush(stdout)\n",
    "end\n",
    "\n",
    "CSV.write(outputvsc, DataFrame(accData));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done: m 500\n",
      "Done: m 1000\n",
      "Done: m 1500\n",
      "Done: m 2000\n",
      "Done: m 2500\n",
      "Done: m 3000\n",
      "Done: m 3500\n",
      "Done: m 4000\n",
      "Done: m 4500\n",
      "Done: m 5000\n",
      "Done: m 5500\n"
     ]
    }
   ],
   "source": [
    "# Quasi newton methods running time comparison, fixing n and varying m, on well-conditioned matrix\n",
    "λ    = 1e-4\n",
    "n    = 50\n",
    "runs = 10\n",
    "samples = 10\n",
    "\n",
    "# output csv\n",
    "outputvsc = joinpath(baseDir, \"Quasi-Newton-Comparison-time-wellcond.csv\");\n",
    "accData = Dict(\n",
    "    :m   => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeLBFGS => Array{Float64}(undef, 0),\n",
    "    :accuracyLBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsLBFGS => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeBFGS => Array{Float64}(undef, 0),\n",
    "    :stdtimeBFGS => Array{Float64}(undef, 0),\n",
    "    :accuracyBFGS => Array{Float64}(undef, 0),\n",
    "    :meanallocsBFGS => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeBFGSDogleg => Array{Float64}(undef, 0),\n",
    "    :stdtimeBFGSDogleg => Array{Float64}(undef, 0),\n",
    "    :accuracyBFGSDogleg => Array{Float64}(undef, 0),\n",
    "    :meanallocsBFGSDogleg => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeDFP => Array{Float64}(undef, 0),\n",
    "    :stdtimeDFP => Array{Float64}(undef, 0),\n",
    "    :accuracyDFP => Array{Float64}(undef, 0),\n",
    "    :meanallocsDFP => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeDFPDogleg => Array{Float64}(undef, 0),\n",
    "    :stdtimeDFPDogleg => Array{Float64}(undef, 0),\n",
    "    :accuracyDFPDogleg => Array{Float64}(undef, 0),\n",
    "    :meanallocsDFPDogleg => Array{Float64}(undef, 0),\n",
    "\n",
    "    :meantimeSR1 => Array{Float64}(undef, 0),\n",
    "    :stdtimeSR1 => Array{Float64}(undef, 0),\n",
    "    :accuracySR1 => Array{Float64}(undef, 0),\n",
    "    :meanallocsSR1 => Array{Float64}(undef, 0)\n",
    "    \n",
    "    )\n",
    "\n",
    "\n",
    "for i ∈ 0:runs\n",
    "    m = 500 + i * 500\n",
    "\n",
    "    gf = genFunc(:randDataset, λ=λ, m=m, n=n)\n",
    "    ls = LeastSquaresF(gf)\n",
    "\n",
    "    t_lbfgs = @benchmark begin\n",
    "        LimitedMemoryBFGS($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_lbfgs = LimitedMemoryBFGS(ls)\n",
    "\n",
    "\n",
    "    BFGS.BFGSorDFP = :BFGS\n",
    "\n",
    "    t_bfgs = @benchmark begin\n",
    "        BroydenFletcherGoldfarbShanno($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_bfgs = BroydenFletcherGoldfarbShanno(ls)\n",
    "\n",
    "    t_bfgs_dogleg = @benchmark begin\n",
    "        BroydenFletcherGoldfarbShannoDogleg($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_bfgs_dogleg = BroydenFletcherGoldfarbShannoDogleg(ls)\n",
    "\n",
    "    BFGS.BFGSorDFP = :DFP\n",
    "\n",
    "    t_dfp = @benchmark begin\n",
    "        BroydenFletcherGoldfarbShanno($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_dfp = BroydenFletcherGoldfarbShanno(ls)\n",
    "\n",
    "    t_dfp_dogleg = @benchmark begin\n",
    "        BroydenFletcherGoldfarbShannoDogleg($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_dfp_dogleg = BroydenFletcherGoldfarbShannoDogleg(ls)\n",
    "\n",
    "    t_sr1 = @benchmark begin\n",
    "        SymmetricRank1($ls)\n",
    "    end samples=samples evals=1\n",
    "    res_sr1 = SymmetricRank1(ls)\n",
    "    \n",
    "    push!(accData[:m], m)\n",
    "    push!(accData[:meantimeLBFGS], mean(t_lbfgs.times))\n",
    "    push!(accData[:stdtimeLBFGS], std(t_lbfgs.times))\n",
    "    push!(accData[:accuracyLBFGS], norm(res_lbfgs.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsLBFGS], mean(t_lbfgs.memory))\n",
    "\n",
    "    push!(accData[:meantimeBFGS], mean(t_bfgs.times))\n",
    "    push!(accData[:stdtimeBFGS], std(t_bfgs.times))\n",
    "    push!(accData[:accuracyBFGS], norm(res_bfgs.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsBFGS], mean(t_bfgs.memory))\n",
    "\n",
    "    push!(accData[:meantimeBFGSDogleg], mean(t_bfgs_dogleg.times))\n",
    "    push!(accData[:stdtimeBFGSDogleg], std(t_bfgs_dogleg.times))\n",
    "    push!(accData[:accuracyBFGSDogleg], norm(res_bfgs_dogleg.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsBFGSDogleg], mean(t_bfgs_dogleg.memory))\n",
    "\n",
    "    push!(accData[:meantimeDFP], mean(t_dfp.times))\n",
    "    push!(accData[:stdtimeDFP], std(t_dfp.times))\n",
    "    push!(accData[:accuracyDFP], norm(res_dfp.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsDFP], mean(t_dfp.memory))\n",
    "\n",
    "    push!(accData[:meantimeDFPDogleg], mean(t_dfp_dogleg.times))\n",
    "    push!(accData[:stdtimeDFPDogleg], std(t_dfp_dogleg.times))\n",
    "    push!(accData[:accuracyDFPDogleg], norm(res_dfp_dogleg.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsDFPDogleg], mean(t_dfp_dogleg.memory))\n",
    "\n",
    "    push!(accData[:meantimeSR1], mean(t_sr1.times))\n",
    "    push!(accData[:stdtimeSR1], std(t_sr1.times))\n",
    "    push!(accData[:accuracySR1], norm(res_sr1.x - gf[:w_star]) / norm(gf[:w_star]))\n",
    "    push!(accData[:meanallocsSR1], mean(t_sr1.memory))\n",
    "    println(\"Done: m \" * string(m))\n",
    "    flush(stdout)\n",
    "end\n",
    "\n",
    "CSV.write(outputvsc, DataFrame(accData));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.9.3",
   "language": "julia",
   "name": "julia-1.9"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.9.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}